[Bugfix][Kernel] Fix batch invariance in RMSNorm kernels by pinning block size (#48391)

Signed-off-by: oops-oom <73481342@qq.com>
Signed-off-by: oops-oom <liubin8905@vip.qq.com>
Co-authored-by: oops-oom <73481342@qq.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
oops-oom
2026-07-28 22:24:02 +08:00
committed by GitHub
co-authored by oops-oom Claude Opus 4.8 Wentao Ye Shengqi Chen
parent 94100b5915
commit b6cbba8bc8
6 changed files with 189 additions and 21 deletions
@@ -2,6 +2,7 @@
#include "../../torch_utils.h"
#include "../../dispatch_utils.h"
#include "../../../core/batch_invariant.hpp"
#include "layernorm_utils.cuh"
#include "quant_conversions.cuh"
@@ -231,7 +232,9 @@ void rms_norm_per_block_quant_dispatch(
auto num_tokens = input.numel() / hidden_size;
dim3 grid(num_tokens);
const int max_block_size = (num_tokens <= 256) ? 512 : 256;
const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
const int max_block_size =
batch_invariant_launch ? 512 : ((num_tokens <= 256) ? 512 : 256);
dim3 block(std::min(hidden_size, max_block_size));
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());