[ROCm][Perf] Optimize DSV4.1 K=512 decode top-k on gfx950 (#56743)
Signed-off-by: fai <fangzhouai@gmail.com>
This commit is contained in:
+205
-58
@@ -154,7 +154,8 @@ __device__ void vectorized_process(size_t thread_rank, size_t num_threads,
|
||||
}
|
||||
|
||||
template <int step, int kNumThreadsPerBlock, int kNumBins, int kNumFinalItems,
|
||||
bool multipleBlocksPerRow, bool mergeBlocks, typename SmemFinalType,
|
||||
bool multipleBlocksPerRow, bool mergeBlocks,
|
||||
bool useTopK512Optimization, typename SmemFinalType,
|
||||
typename SmemOutputType>
|
||||
__device__ bool processHistogramStep(
|
||||
const int* indices, const float* logits, int rowEnd, uint32_t& logitPattern,
|
||||
@@ -180,10 +181,15 @@ __device__ bool processHistogramStep(
|
||||
<< patternShift;
|
||||
}
|
||||
|
||||
int negativeInfinityCount = 0;
|
||||
auto distributeToBins = [&](float logit, int /* idx */ = 0) {
|
||||
if (isPartialMatch<patternShift>(logit, logitPattern)) {
|
||||
uint32_t binIdx = extractBinIdx<step>(logit);
|
||||
atomicAdd(&smemFinal.histo.data[binIdx], 1);
|
||||
if (useTopK512Optimization && __float_as_uint(logit) == 0xff800000u) {
|
||||
++negativeInfinityCount;
|
||||
} else {
|
||||
atomicAdd(&smemFinal.histo.data[binIdx], 1);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -198,6 +204,12 @@ __device__ bool processHistogramStep(
|
||||
distributeToBins(logit, idx);
|
||||
}
|
||||
}
|
||||
if constexpr (useTopK512Optimization) {
|
||||
if (negativeInfinityCount > 0) {
|
||||
const auto bin = extractBinIdx<step>(-INFINITY);
|
||||
atomicAdd(&smemFinal.histo.data[bin], negativeInfinityCount);
|
||||
}
|
||||
}
|
||||
// Make sure the histogram is ready.
|
||||
__syncthreads();
|
||||
|
||||
@@ -254,6 +266,12 @@ __device__ bool processHistogramStep(
|
||||
|
||||
// The threshold bin.
|
||||
thresholdBinIdx = smemThresholdBinIdx[0];
|
||||
// Resolve a -inf cutoff with exact tie emission, keeping sort padding out.
|
||||
const bool finalBinFits =
|
||||
smemFinalBinSize[0] <= kNumFinalItems &&
|
||||
!(useTopK512Optimization && step < 3 &&
|
||||
isPartialMatch<patternShift>(-INFINITY, logitPattern) &&
|
||||
thresholdBinIdx == extractBinIdx<step>(-INFINITY));
|
||||
|
||||
auto processBins = [&](float logit, int idx) {
|
||||
if (isPartialMatch<patternShift>(logit, logitPattern)) {
|
||||
@@ -262,8 +280,7 @@ __device__ bool processHistogramStep(
|
||||
// 1. This is step 0 and the threshold bin is small enough (no step 1)
|
||||
// 2. This is step >= 1 (where pattern matching filters correctly)
|
||||
// This prevents duplicates when step 0 and step 1 both run.
|
||||
bool shouldWriteDirectly =
|
||||
(step == 0 && smemFinalBinSize[0] <= kNumFinalItems) || (step >= 1);
|
||||
bool shouldWriteDirectly = (step == 0 && finalBinFits) || (step >= 1);
|
||||
if (binIdx < thresholdBinIdx && shouldWriteDirectly) {
|
||||
// The element is part of the top-k selection
|
||||
int dstIdx = atomicAdd(&smemFoundTopKValues[0], 1);
|
||||
@@ -279,8 +296,7 @@ __device__ bool processHistogramStep(
|
||||
}
|
||||
if constexpr (step < 3) {
|
||||
// Only fill the final items for sorting if the threshold bin fits
|
||||
if (binIdx == thresholdBinIdx &&
|
||||
smemFinalBinSize[0] <= kNumFinalItems) {
|
||||
if (binIdx == thresholdBinIdx && finalBinFits) {
|
||||
int dstIdx = atomicAdd(&smemFinalDstIdx[0], 1);
|
||||
smemFinal.items.logits[dstIdx] = logit;
|
||||
if constexpr (mergeBlocks) {
|
||||
@@ -325,20 +341,21 @@ __device__ bool processHistogramStep(
|
||||
__syncthreads();
|
||||
|
||||
// Check if we should continue to next step
|
||||
return smemFinalBinSize[0] > kNumFinalItems;
|
||||
return !finalBinFits;
|
||||
}
|
||||
|
||||
// Follows half - 11 - 11 - 10 bit iterations
|
||||
// Keep the adaptive kernel's device instantiation separate from legacy calls.
|
||||
template <int kNumThreadsPerBlock, int kNumBins, bool useRadixSort,
|
||||
bool multipleBlocksPerRow = false, bool mergeBlocks = false,
|
||||
bool deviceLengthAware = false>
|
||||
bool deviceLengthAware = false, bool useTopK512Optimization = false>
|
||||
static __device__ void topKPerRowJob(const int* indices, const float* logits,
|
||||
int rowStart, int rowEnd, int* outIndices,
|
||||
float* outLogits, int stride1, int topK) {
|
||||
static_assert(!deviceLengthAware || multipleBlocksPerRow != mergeBlocks);
|
||||
static_assert(!useTopK512Optimization || kNumThreadsPerBlock == 1024);
|
||||
// The number of slots for the final pass.
|
||||
static constexpr int kNumFinalItems = 2048;
|
||||
static constexpr int kNumFinalItems = useTopK512Optimization ? 1024 : 2048;
|
||||
// The number of elements per thread for the final sort.
|
||||
static constexpr int kNumFinalItemsPerThread =
|
||||
kNumFinalItems / kNumThreadsPerBlock;
|
||||
@@ -404,7 +421,7 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
|
||||
rowIt += kNumThreadsPerBlock) {
|
||||
outIndices[rowIt] = -1;
|
||||
if constexpr (multipleBlocksPerRow) {
|
||||
outLogits[rowIt] = -FLT_MAX;
|
||||
outLogits[rowIt] = useTopK512Optimization ? -INFINITY : -FLT_MAX;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -422,7 +439,8 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
|
||||
// Step 0: Process first 11 bits of half representation
|
||||
bool continueToNextStep =
|
||||
processHistogramStep<0, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
|
||||
multipleBlocksPerRow, mergeBlocks>(
|
||||
multipleBlocksPerRow, mergeBlocks,
|
||||
useTopK512Optimization>(
|
||||
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
|
||||
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
|
||||
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
|
||||
@@ -431,7 +449,8 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
|
||||
// Step 1: Process next 11 bits
|
||||
continueToNextStep =
|
||||
processHistogramStep<1, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
|
||||
multipleBlocksPerRow, mergeBlocks>(
|
||||
multipleBlocksPerRow, mergeBlocks,
|
||||
useTopK512Optimization>(
|
||||
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
|
||||
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
|
||||
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
|
||||
@@ -441,7 +460,8 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
|
||||
// Step 2: Process next 11 bits
|
||||
continueToNextStep =
|
||||
processHistogramStep<2, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
|
||||
multipleBlocksPerRow, mergeBlocks>(
|
||||
multipleBlocksPerRow, mergeBlocks,
|
||||
useTopK512Optimization>(
|
||||
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
|
||||
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
|
||||
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
|
||||
@@ -450,7 +470,8 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
|
||||
if (continueToNextStep) {
|
||||
// Step 3: Process last 10 bits
|
||||
processHistogramStep<3, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
|
||||
multipleBlocksPerRow, mergeBlocks>(
|
||||
multipleBlocksPerRow, mergeBlocks,
|
||||
useTopK512Optimization>(
|
||||
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
|
||||
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
|
||||
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
|
||||
@@ -460,50 +481,7 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
|
||||
// The histogram did not proceed to the final 10 bits, therefore we need to
|
||||
// sort the final items The logits of the elements to be sorted in the final
|
||||
// pass.
|
||||
if constexpr (useRadixSort) {
|
||||
// Sorting with radix sort
|
||||
float finalLogits[kNumFinalItemsPerThread];
|
||||
// The indices of the elements to be sorted in the final pass.
|
||||
int finalIndices[kNumFinalItemsPerThread];
|
||||
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
|
||||
finalLogits[ii] = -FLT_MAX;
|
||||
}
|
||||
|
||||
// Read the elements from SMEM.
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
|
||||
int srcIdx = ii * kNumThreadsPerBlock + threadIdx.x;
|
||||
if (srcIdx < smemFinalDstIdx[0]) {
|
||||
finalLogits[ii] = smemFinal.items.logits[srcIdx];
|
||||
finalIndices[ii] = smemFinal.items.indices[srcIdx];
|
||||
}
|
||||
}
|
||||
// Make sure the shared memory has been read.
|
||||
__syncthreads();
|
||||
|
||||
// Sort the elements.
|
||||
FinalSort(smemFinal.finalSort)
|
||||
.SortDescendingBlockedToStriped(finalLogits, finalIndices);
|
||||
|
||||
// Copy the data back to the shared memory storage.
|
||||
int baseIdx = smemFoundTopKValues[0];
|
||||
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
|
||||
int srcIdx = ii * kNumThreadsPerBlock + threadIdx.x;
|
||||
int dstIdx = baseIdx + srcIdx;
|
||||
|
||||
if (dstIdx < topK) {
|
||||
smemOutput[dstIdx] = finalIndices[ii];
|
||||
if constexpr (multipleBlocksPerRow) {
|
||||
reinterpret_cast<float*>(smemOutput + topK)[dstIdx] =
|
||||
finalLogits[ii];
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
auto insertionSort = [&]() {
|
||||
// Sorting with insertion sort
|
||||
auto baseIdx = smemFoundTopKValues[0];
|
||||
for (int i = threadIdx.x; i < smemFinalDstIdx[0];
|
||||
@@ -525,6 +503,57 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
if constexpr (useRadixSort) {
|
||||
if (useTopK512Optimization && smemFinalDstIdx[0] <= 128) {
|
||||
insertionSort();
|
||||
} else {
|
||||
// Sorting with radix sort
|
||||
float finalLogits[kNumFinalItemsPerThread];
|
||||
// The indices of the elements to be sorted in the final pass.
|
||||
int finalIndices[kNumFinalItemsPerThread];
|
||||
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
|
||||
finalLogits[ii] = useTopK512Optimization ? -INFINITY : -FLT_MAX;
|
||||
if constexpr (useTopK512Optimization) finalIndices[ii] = -1;
|
||||
}
|
||||
|
||||
// Read the elements from SMEM.
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
|
||||
int srcIdx = ii * kNumThreadsPerBlock + threadIdx.x;
|
||||
if (srcIdx < smemFinalDstIdx[0]) {
|
||||
finalLogits[ii] = smemFinal.items.logits[srcIdx];
|
||||
finalIndices[ii] = smemFinal.items.indices[srcIdx];
|
||||
}
|
||||
}
|
||||
// Make sure the shared memory has been read.
|
||||
__syncthreads();
|
||||
|
||||
// Sort the elements.
|
||||
FinalSort(smemFinal.finalSort)
|
||||
.SortDescendingBlockedToStriped(finalLogits, finalIndices);
|
||||
|
||||
// Copy the data back to the shared memory storage.
|
||||
int baseIdx = smemFoundTopKValues[0];
|
||||
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
|
||||
int srcIdx = ii * kNumThreadsPerBlock + threadIdx.x;
|
||||
int dstIdx = baseIdx + srcIdx;
|
||||
|
||||
if (dstIdx < topK) {
|
||||
smemOutput[dstIdx] = finalIndices[ii];
|
||||
if constexpr (multipleBlocksPerRow) {
|
||||
reinterpret_cast<float*>(smemOutput + topK)[dstIdx] =
|
||||
finalLogits[ii];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
insertionSort();
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
@@ -669,6 +698,70 @@ __launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecodeDeviceLengthAware(
|
||||
stride1, topK);
|
||||
}
|
||||
|
||||
#ifdef USE_ROCM
|
||||
// The grid and workspace stay fixed while graph replays change device lengths.
|
||||
__device__ int gfx950TopK512ActiveBlocks(int rowLength, int numRows,
|
||||
int maxBlocks) {
|
||||
int activeBlocks = 1;
|
||||
if (numRows <= 32 && rowLength > 16 * 1024) {
|
||||
activeBlocks = rowLength <= 128 * 1024 ? 8
|
||||
: rowLength <= 256 * 1024 ? 10
|
||||
: 16;
|
||||
} else if (numRows <= 128 && rowLength > 64 * 1024) {
|
||||
activeBlocks = rowLength <= 512 * 1024 ? 4 : 10;
|
||||
} else if (numRows > 128 && numRows <= 256 && rowLength > 256 * 1024) {
|
||||
activeBlocks = numRows <= 160 ? 3 : 2;
|
||||
}
|
||||
// Every block in a merged row must contain at least k real indices.
|
||||
return min(min(activeBlocks, maxBlocks), max(1, rowLength / 512));
|
||||
}
|
||||
|
||||
template <int kNumThreadsPerBlock, bool mergeBlocks = false>
|
||||
static __global__
|
||||
__launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecode512DeviceLengthAware(
|
||||
const float* logits, const int* seqLens, int* outIndices,
|
||||
int* outIndicesAux, float* outLogitsAux, int stride0, int next_n,
|
||||
int seqLensIs2D, int maxBlocks) {
|
||||
#if defined(__gfx950__)
|
||||
constexpr int topK = 512;
|
||||
constexpr int kNumBins = 2048;
|
||||
const int rowIdx = blockIdx.x;
|
||||
const int rowEnd =
|
||||
seqLensIs2D
|
||||
? max(0, seqLens[rowIdx])
|
||||
: max(0, seqLens[rowIdx / next_n] - next_n + rowIdx % next_n + 1);
|
||||
const int activeBlocks =
|
||||
gfx950TopK512ActiveBlocks(rowEnd, gridDim.x, maxBlocks);
|
||||
outIndices += static_cast<int64_t>(rowIdx) * topK;
|
||||
if constexpr (mergeBlocks) {
|
||||
if (activeBlocks == 1) return;
|
||||
const int64_t offset = static_cast<int64_t>(rowIdx) * maxBlocks * topK;
|
||||
topKPerRowJob<kNumThreadsPerBlock, kNumBins, true, false, true, false,
|
||||
true>(outIndicesAux + offset, outLogitsAux + offset, 0,
|
||||
activeBlocks * topK, outIndices, nullptr, 1, topK);
|
||||
} else {
|
||||
if (blockIdx.y >= activeBlocks) return;
|
||||
logits += static_cast<int64_t>(rowIdx) * stride0;
|
||||
if (activeBlocks == 1) {
|
||||
topKPerRowJob<kNumThreadsPerBlock, kNumBins, true, false, false, false,
|
||||
true>(nullptr, logits, 0, rowEnd, outIndices, nullptr, 1,
|
||||
topK);
|
||||
return;
|
||||
}
|
||||
const int blockSize = rowEnd / activeBlocks;
|
||||
const int rowStart = blockSize * blockIdx.y;
|
||||
const int blockRowEnd =
|
||||
blockIdx.y + 1 == activeBlocks ? rowEnd : rowStart + blockSize;
|
||||
const int64_t offset =
|
||||
(static_cast<int64_t>(rowIdx) * maxBlocks + blockIdx.y) * topK;
|
||||
topKPerRowJob<kNumThreadsPerBlock, kNumBins, true, true, false, false,
|
||||
true>(nullptr, logits, rowStart, blockRowEnd,
|
||||
outIndicesAux + offset, outLogitsAux + offset, 1, topK);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
void apply_repetition_penalties_(
|
||||
@@ -731,6 +824,60 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
|
||||
// the same seq_len and the kernel computes the per-row offset itself.
|
||||
int seqLensIs2D = seqLens.dim() == 2 ? 1 : 0;
|
||||
|
||||
#ifdef USE_ROCM
|
||||
const bool useGfx950DeviceLengthAwareTopK512 =
|
||||
topK == 512 && stride1 == 1 && numRows > 0 && numRows <= 384 &&
|
||||
numColumns <= 1024 * 1024 &&
|
||||
std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0;
|
||||
if (useGfx950DeviceLengthAwareTopK512) {
|
||||
int multipleBlocksPerRowConfig = 1;
|
||||
if (numRows <= 32 && numColumns > 16 * 1024) {
|
||||
multipleBlocksPerRowConfig = 16;
|
||||
} else if (numRows <= 128 && numColumns > 64 * 1024) {
|
||||
multipleBlocksPerRowConfig = 10;
|
||||
} else if (numRows <= 160 && numColumns > 256 * 1024) {
|
||||
multipleBlocksPerRowConfig = 3;
|
||||
} else if (numRows <= 256 && numColumns > 256 * 1024) {
|
||||
multipleBlocksPerRowConfig = 2;
|
||||
}
|
||||
|
||||
constexpr int kNumThreadsPerSplitBlock = 1024;
|
||||
if (multipleBlocksPerRowConfig == 1) {
|
||||
vllm::topKPerRowDecode512DeviceLengthAware<kNumThreadsPerSplitBlock>
|
||||
<<<numRows, kNumThreadsPerSplitBlock, 2 * topK * sizeof(int32_t),
|
||||
stream>>>(logits.const_data_ptr<float>(),
|
||||
seqLens.const_data_ptr<int>(),
|
||||
indices.mutable_data_ptr<int>(), nullptr, nullptr,
|
||||
static_cast<int>(stride0), static_cast<int>(next_n),
|
||||
seqLensIs2D, multipleBlocksPerRowConfig);
|
||||
return;
|
||||
}
|
||||
const auto outIndicesAux = torch::stable::empty(
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
torch::headeronly::ScalarType::Int, std::nullopt, logits.device());
|
||||
const auto outLogitsAux = torch::stable::empty(
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
torch::headeronly::ScalarType::Float, std::nullopt, logits.device());
|
||||
vllm::topKPerRowDecode512DeviceLengthAware<kNumThreadsPerSplitBlock>
|
||||
<<<dim3(numRows, multipleBlocksPerRowConfig), kNumThreadsPerSplitBlock,
|
||||
2 * topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
indices.mutable_data_ptr<int>(),
|
||||
outIndicesAux.mutable_data_ptr<int>(),
|
||||
outLogitsAux.mutable_data_ptr<float>(), static_cast<int>(stride0),
|
||||
static_cast<int>(next_n), seqLensIs2D, multipleBlocksPerRowConfig);
|
||||
constexpr int kNumThreadsPerBlockMerge = 1024;
|
||||
vllm::topKPerRowDecode512DeviceLengthAware<kNumThreadsPerBlockMerge, true>
|
||||
<<<numRows, kNumThreadsPerBlockMerge, topK * sizeof(int32_t), stream>>>(
|
||||
nullptr, seqLens.const_data_ptr<int>(),
|
||||
indices.mutable_data_ptr<int>(),
|
||||
outIndicesAux.mutable_data_ptr<int>(),
|
||||
outLogitsAux.mutable_data_ptr<float>(), 0, static_cast<int>(next_n),
|
||||
seqLensIs2D, multipleBlocksPerRowConfig);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (numColumns < kSortingAlgorithmThreshold) {
|
||||
// Use insertion sort
|
||||
vllm::topKPerRowDecode<kNumThreadsPerBlock, false>
|
||||
|
||||
Reference in New Issue
Block a user