[Bugfix][Kernel] Fix batch invariance in RMSNorm kernels by pinning block size (#48391)
Signed-off-by: oops-oom <73481342@qq.com> Signed-off-by: oops-oom <liubin8905@vip.qq.com> Co-authored-by: oops-oom <73481342@qq.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
co-authored by
oops-oom
Claude Opus 4.8
Wentao Ye
Shengqi Chen
parent
94100b5915
commit
b6cbba8bc8
@@ -249,7 +249,9 @@ void rms_norm(torch::stable::Tensor& out, // [..., hidden_size]
|
||||
int64_t input_shape_d3 = (num_dims >= 4) ? input.size(-3) : 0;
|
||||
|
||||
// For large num_tokens, use smaller blocks to increase SM concurrency.
|
||||
const int max_block_size = (num_tokens < 256) ? 1024 : 256;
|
||||
const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
|
||||
const int max_block_size =
|
||||
batch_invariant_launch ? 1024 : ((num_tokens < 256) ? 1024 : 256);
|
||||
dim3 grid(num_tokens);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
@@ -325,8 +327,13 @@ void fused_add_rms_norm(torch::stable::Tensor& input, // [..., hidden_size]
|
||||
/* This kernel is memory-latency bound in many scenarios.
|
||||
When num_tokens is large, a smaller block size allows
|
||||
for increased block occupancy on CUs and better latency
|
||||
hiding on global mem ops. */
|
||||
const int max_block_size = (num_tokens < 256) ? 1024 : 256;
|
||||
hiding on global mem ops. In batch-invariant mode the block size must
|
||||
not depend on num_tokens, otherwise the same token would use a different
|
||||
reduction width (and thus a different floating-point summation order)
|
||||
across batches; lock it to 1024 to keep results bit-exact. */
|
||||
const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
|
||||
const int max_block_size =
|
||||
batch_invariant_launch ? 1024 : ((num_tokens < 256) ? 1024 : 256);
|
||||
dim3 block(std::min(hidden_size, max_block_size));
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
@@ -337,7 +344,6 @@ void fused_add_rms_norm(torch::stable::Tensor& input, // [..., hidden_size]
|
||||
auto res_ptr = reinterpret_cast<std::uintptr_t>(residual.data_ptr());
|
||||
bool offsets_are_multiple_of_vector_width =
|
||||
hidden_size % vector_width == 0 && input_stride % vector_width == 0;
|
||||
bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
|
||||
const bool has_weight = weight.has_value();
|
||||
if (has_weight) {
|
||||
auto wt_ptr = reinterpret_cast<std::uintptr_t>(weight->data_ptr());
|
||||
|
||||
@@ -215,7 +215,9 @@ void rms_norm_static_fp8_quant(
|
||||
int num_tokens = input.numel() / hidden_size;
|
||||
|
||||
// For large num_tokens, use smaller blocks to increase SM concurrency.
|
||||
const int max_block_size = (num_tokens < 256) ? 1024 : 256;
|
||||
const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
|
||||
const int max_block_size =
|
||||
batch_invariant_launch ? 1024 : ((num_tokens < 256) ? 1024 : 256);
|
||||
dim3 grid(num_tokens);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
@@ -279,7 +281,9 @@ void fused_add_rms_norm_static_fp8_quant(
|
||||
When num_tokens is large, a smaller block size allows
|
||||
for increased block occupancy on CUs and better latency
|
||||
hiding on global mem ops. */
|
||||
const int max_block_size = (num_tokens < 256) ? 1024 : 256;
|
||||
const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
|
||||
const int max_block_size =
|
||||
batch_invariant_launch ? 1024 : ((num_tokens < 256) ? 1024 : 256);
|
||||
dim3 block(std::min(hidden_size, max_block_size));
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
@@ -296,7 +300,6 @@ void fused_add_rms_norm_static_fp8_quant(
|
||||
auto wt_ptr = reinterpret_cast<std::uintptr_t>(weight.data_ptr());
|
||||
bool ptrs_are_aligned =
|
||||
inp_ptr % 16 == 0 && res_ptr % 16 == 0 && wt_ptr % 16 == 0;
|
||||
bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
|
||||
if (ptrs_are_aligned && hidden_size % 8 == 0 && input_stride % 8 == 0 &&
|
||||
!batch_invariant_launch) {
|
||||
LAUNCH_FUSED_ADD_RMS_NORM(8);
|
||||
|
||||
+4
-1
@@ -2,6 +2,7 @@
|
||||
#include "../../torch_utils.h"
|
||||
|
||||
#include "../../dispatch_utils.h"
|
||||
#include "../../../core/batch_invariant.hpp"
|
||||
#include "layernorm_utils.cuh"
|
||||
#include "quant_conversions.cuh"
|
||||
|
||||
@@ -231,7 +232,9 @@ void rms_norm_per_block_quant_dispatch(
|
||||
auto num_tokens = input.numel() / hidden_size;
|
||||
|
||||
dim3 grid(num_tokens);
|
||||
const int max_block_size = (num_tokens <= 256) ? 512 : 256;
|
||||
const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
|
||||
const int max_block_size =
|
||||
batch_invariant_launch ? 512 : ((num_tokens <= 256) ? 512 : 256);
|
||||
dim3 block(std::min(hidden_size, max_block_size));
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
|
||||
Reference in New Issue
Block a user