[ROCm][Perf] Optimize DeepSeek V4 C4A top-k with AITER (#52882)
This commit is contained in:
+132
-11
@@ -2,6 +2,8 @@
|
||||
#include "dispatch_utils.h"
|
||||
#include "torch_utils.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include <cub/cub.cuh>
|
||||
#else
|
||||
@@ -327,11 +329,14 @@ __device__ bool processHistogramStep(
|
||||
}
|
||||
|
||||
// Follows half - 11 - 11 - 10 bit iterations
|
||||
// Keep the adaptive kernel's device instantiation separate from legacy calls.
|
||||
template <int kNumThreadsPerBlock, int kNumBins, bool useRadixSort,
|
||||
bool multipleBlocksPerRow = false, bool mergeBlocks = false>
|
||||
bool multipleBlocksPerRow = false, bool mergeBlocks = false,
|
||||
bool deviceLengthAware = false>
|
||||
static __device__ void topKPerRowJob(const int* indices, const float* logits,
|
||||
int rowStart, int rowEnd, int* outIndices,
|
||||
float* outLogits, int stride1, int topK) {
|
||||
static_assert(!deviceLengthAware || multipleBlocksPerRow != mergeBlocks);
|
||||
// The number of slots for the final pass.
|
||||
static constexpr int kNumFinalItems = 2048;
|
||||
// The number of elements per thread for the final sort.
|
||||
@@ -613,6 +618,57 @@ static __global__ __launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecode(
|
||||
indices, logits, rowStart, rowEnd, outIndices, outLogits, stride1, topK);
|
||||
}
|
||||
|
||||
// Measured native-kernel policy for gfx950, FP32, and k=1024. This
|
||||
// active-column split crossover is independent of the Python AITER/native
|
||||
// backend crossover; the two policies can be retuned separately. At exactly
|
||||
// 64K, an eager call may select AITER while a captured native replay uses two
|
||||
// splits.
|
||||
constexpr int kGfx950C4AShortRowNumBlocks = 3;
|
||||
constexpr int kGfx950C4ALongRowNumBlocks = 2;
|
||||
constexpr int kGfx950C4ATwoSplitMinRowLength = 64 * 1024;
|
||||
|
||||
template <int kNumThreadsPerBlock, bool mergeBlocks = false>
|
||||
static __global__
|
||||
__launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecodeDeviceLengthAware(
|
||||
const float* logits, const int* seqLens, int* outIndices, int stride0,
|
||||
int stride1, const int topK, int next_n, int seqLensIs2D = 0,
|
||||
float* outLogits = nullptr, const int* indices = nullptr) {
|
||||
static constexpr int kNumBins = 2048;
|
||||
int rowIdx = blockIdx.x;
|
||||
int batch_idx = rowIdx / next_n;
|
||||
int next_n_idx = rowIdx % next_n;
|
||||
int seq_len = seqLensIs2D ? seqLens[rowIdx] : seqLens[batch_idx];
|
||||
int rowEnd =
|
||||
seqLensIs2D ? max(0, seq_len) : max(0, seq_len - next_n + next_n_idx + 1);
|
||||
const int activeBlocks = rowEnd >= kGfx950C4ATwoSplitMinRowLength
|
||||
? kGfx950C4ALongRowNumBlocks
|
||||
: kGfx950C4AShortRowNumBlocks;
|
||||
int rowStart = 0;
|
||||
|
||||
if constexpr (!mergeBlocks) {
|
||||
if (blockIdx.y >= activeBlocks) return;
|
||||
const auto blockSize = rowEnd / activeBlocks;
|
||||
rowStart = blockSize * blockIdx.y;
|
||||
rowEnd = activeBlocks == blockIdx.y + 1 ? rowEnd : rowStart + blockSize;
|
||||
outIndices +=
|
||||
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK +
|
||||
blockIdx.y * topK;
|
||||
outLogits +=
|
||||
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK +
|
||||
blockIdx.y * topK;
|
||||
} else {
|
||||
rowEnd = activeBlocks * topK;
|
||||
indices +=
|
||||
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK;
|
||||
outIndices += static_cast<int64_t>(rowIdx) * topK;
|
||||
}
|
||||
logits += static_cast<int64_t>(rowIdx) * stride0;
|
||||
|
||||
topKPerRowJob<kNumThreadsPerBlock, kNumBins, true, !mergeBlocks, mergeBlocks,
|
||||
true>(indices, logits, rowStart, rowEnd, outIndices, outLogits,
|
||||
stride1, topK);
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
void apply_repetition_penalties_(
|
||||
@@ -693,7 +749,60 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
|
||||
static_cast<int>(next_n), seqLensIs2D);
|
||||
} else {
|
||||
// Long sequences are run in two steps
|
||||
constexpr auto multipleBlocksPerRowConfig = 10;
|
||||
#ifdef USE_ROCM
|
||||
const bool useGfx950DeviceLengthAwareTopK1024 =
|
||||
topK == 1024 && numRows > 64 && numRows <= 256 &&
|
||||
std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0;
|
||||
if (useGfx950DeviceLengthAwareTopK1024) {
|
||||
// Keep three blocks for short FULL-graph replays, but let each row use
|
||||
// two blocks once its live device length reaches the long-context
|
||||
// crossover. The grid and workspace shapes remain capture-stable.
|
||||
constexpr int multipleBlocksPerRowConfig =
|
||||
vllm::kGfx950C4AShortRowNumBlocks;
|
||||
const auto outIndicesAux = torch::stable::empty(
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
torch::headeronly::ScalarType::Int, std::nullopt, logits.device());
|
||||
const auto outLogitsAux = torch::stable::empty(
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
torch::headeronly::ScalarType::Float, std::nullopt, logits.device());
|
||||
|
||||
constexpr int kNumThreadsPerSplitBlock = 1024;
|
||||
vllm::topKPerRowDecodeDeviceLengthAware<kNumThreadsPerSplitBlock>
|
||||
<<<dim3(numRows, multipleBlocksPerRowConfig),
|
||||
kNumThreadsPerSplitBlock, 2 * topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D,
|
||||
outLogitsAux.mutable_data_ptr<float>());
|
||||
constexpr int kNumThreadsPerBlockMerge = 1024;
|
||||
vllm::topKPerRowDecodeDeviceLengthAware<kNumThreadsPerBlockMerge, true>
|
||||
<<<numRows, kNumThreadsPerBlockMerge, topK * sizeof(int32_t),
|
||||
stream>>>(
|
||||
outLogitsAux.const_data_ptr<float>(),
|
||||
seqLens.const_data_ptr<int>(), indices.mutable_data_ptr<int>(),
|
||||
multipleBlocksPerRowConfig * topK, 1, static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D, nullptr,
|
||||
outIndicesAux.const_data_ptr<int>());
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef USE_ROCM
|
||||
const bool useGfx950TopK1024 =
|
||||
topK == 1024 && numRows <= 64 &&
|
||||
std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0;
|
||||
#else
|
||||
constexpr bool useGfx950TopK1024 = false;
|
||||
#endif
|
||||
int multipleBlocksPerRowConfig = 10;
|
||||
if (useGfx950TopK1024) {
|
||||
if (numRows <= 32) {
|
||||
multipleBlocksPerRowConfig = 8;
|
||||
} else {
|
||||
multipleBlocksPerRowConfig = 4;
|
||||
}
|
||||
}
|
||||
|
||||
const auto outIndicesAux = torch::stable::empty(
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
@@ -702,14 +811,26 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
torch::headeronly::ScalarType::Float, std::nullopt, logits.device());
|
||||
|
||||
vllm::topKPerRowDecode<kNumThreadsPerBlock, true, true>
|
||||
<<<dim3(numRows, multipleBlocksPerRowConfig), kNumThreadsPerBlock,
|
||||
2 * topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D,
|
||||
outLogitsAux.mutable_data_ptr<float>());
|
||||
if (useGfx950TopK1024) {
|
||||
constexpr int kNumThreadsPerSplitBlock = 1024;
|
||||
vllm::topKPerRowDecode<kNumThreadsPerSplitBlock, true, true>
|
||||
<<<dim3(numRows, multipleBlocksPerRowConfig),
|
||||
kNumThreadsPerSplitBlock, 2 * topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D,
|
||||
outLogitsAux.mutable_data_ptr<float>());
|
||||
} else {
|
||||
vllm::topKPerRowDecode<kNumThreadsPerBlock, true, true>
|
||||
<<<dim3(numRows, multipleBlocksPerRowConfig), kNumThreadsPerBlock,
|
||||
2 * topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D,
|
||||
outLogitsAux.mutable_data_ptr<float>());
|
||||
}
|
||||
|
||||
constexpr int kNumThreadsPerBlockMerge = 1024;
|
||||
vllm::topKPerRowDecode<kNumThreadsPerBlockMerge, true, false, true>
|
||||
@@ -753,4 +874,4 @@ void top_k_per_row_prefill(const torch::stable::Tensor& logits,
|
||||
static_cast<int>(stride0), static_cast<int>(stride1),
|
||||
static_cast<int>(topK), kSortingAlgorithmThreshold);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user