[Bugfix][Kernel] Fix batch invariance in RMSNorm kernels by pinning block size (#48391)
Signed-off-by: oops-oom <73481342@qq.com> Signed-off-by: oops-oom <liubin8905@vip.qq.com> Co-authored-by: oops-oom <73481342@qq.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
co-authored by
oops-oom
Claude Opus 4.8
Wentao Ye
Shengqi Chen
parent
94100b5915
commit
b6cbba8bc8
+4
-1
@@ -2,6 +2,7 @@
|
||||
#include "../../torch_utils.h"
|
||||
|
||||
#include "../../dispatch_utils.h"
|
||||
#include "../../../core/batch_invariant.hpp"
|
||||
#include "layernorm_utils.cuh"
|
||||
#include "quant_conversions.cuh"
|
||||
|
||||
@@ -231,7 +232,9 @@ void rms_norm_per_block_quant_dispatch(
|
||||
auto num_tokens = input.numel() / hidden_size;
|
||||
|
||||
dim3 grid(num_tokens);
|
||||
const int max_block_size = (num_tokens <= 256) ? 512 : 256;
|
||||
const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
|
||||
const int max_block_size =
|
||||
batch_invariant_launch ? 512 : ((num_tokens <= 256) ? 512 : 256);
|
||||
dim3 block(std::min(hidden_size, max_block_size));
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
|
||||
Reference in New Issue
Block a user