[Bugfix][Kernel] Fix batch invariance in RMSNorm kernels by pinning block size (#48391)

Signed-off-by: oops-oom <73481342@qq.com>
Signed-off-by: oops-oom <liubin8905@vip.qq.com>
Co-authored-by: oops-oom <73481342@qq.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
oops-oom
2026-07-28 22:24:02 +08:00
committed by GitHub
co-authored by oops-oom Claude Opus 4.8 Wentao Ye Shengqi Chen
parent 94100b5915
commit b6cbba8bc8
6 changed files with 189 additions and 21 deletions
@@ -27,8 +27,10 @@ from vllm.platforms import current_platform
"backend",
BACKENDS,
)
@pytest.mark.parametrize("rms_norm_impl", ["default", "vllm_c"])
def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
backend,
rms_norm_impl,
):
"""
Ensures that the same request (the 'needle' prompt) yields identical output
@@ -60,6 +62,16 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
random.seed(seed)
attention_config = {"backend": backend}
# Force the C++ RMSNorm implementation so we actually exercise the
# num_tokens-dependent block-size branches.
kernel_config = None
if rms_norm_impl == "vllm_c":
kernel_config = {
"ir_op_priority": {
"rms_norm": ["vllm_c"],
"fused_add_rms_norm": ["vllm_c"],
}
}
# Allow overrides from environment (useful for CI tuning)
# "facebook/opt-125m" is too small, doesn't reliably test determinism
model = TEST_MODEL
@@ -96,6 +108,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
gpu_memory_utilization=gpu_mem_util,
max_model_len=max_model_len,
attention_config=attention_config,
kernel_config=kernel_config,
)
# Baseline generation for the needle prompt alone.
@@ -923,11 +936,15 @@ def LLM_with_max_seqs(
gpu_memory_utilization: float,
max_model_len: int,
attention_config: dict | None = None,
kernel_config: dict | None = None,
) -> LLM:
"""
Helper to construct an LLM with a specific max_num_seqs (batch-size limit)
using the high-level v1 LLM API, while constraining memory usage.
"""
extra_kwargs: dict = {}
if kernel_config is not None:
extra_kwargs["kernel_config"] = kernel_config
return LLM(
model=model,
max_num_seqs=max_num_seqs,
@@ -939,4 +956,5 @@ def LLM_with_max_seqs(
attention_config=attention_config,
# Enable for MOE models
# enable_expert_parallel=True,
**extra_kwargs,
)
@@ -28,16 +28,18 @@ def _rms_norm_reference(
@skip_if_not_cuda
@pytest.mark.parametrize("batch_size", [1, 4, 16, 64])
@pytest.mark.parametrize("batch_size", [1, 4, 64, 300])
@pytest.mark.parametrize("hidden_size", [512, 2048, 4096, 8192])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("eps", [1e-6, 1e-5])
@pytest.mark.parametrize("seed", list(range(4)))
def test_rms_norm_batch_invariant_vs_reference(
default_vllm_config,
batch_size: int,
hidden_size: int,
dtype: torch.dtype,
eps: float,
seed: int,
):
"""
Compare batch-invariant Triton RMS norm against a PyTorch reference.
@@ -48,7 +50,7 @@ def test_rms_norm_batch_invariant_vs_reference(
device = torch.device(DEVICE_TYPE)
# Create test input and weight
torch.manual_seed(42)
torch.manual_seed(seed)
input_tensor = torch.randn(batch_size, hidden_size, dtype=dtype, device=device)
weight = torch.randn(hidden_size, dtype=dtype, device=device)
@@ -71,7 +73,7 @@ def test_rms_norm_batch_invariant_vs_reference(
atol=atol,
msg=f"RMS norm mismatch for batch_size={batch_size}, "
f"hidden_size={hidden_size}, "
f"dtype={dtype}, eps={eps}",
f"dtype={dtype}, eps={eps}, seed={seed}",
)
@@ -79,17 +81,21 @@ def test_rms_norm_batch_invariant_vs_reference(
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("eps", [1e-6])
@pytest.mark.parametrize("n_extra", [3, 299])
@pytest.mark.parametrize("seed", list(range(16)))
def test_fused_add_rms_norm_batch_invariant_residual_path(
hidden_size: int,
dtype: torch.dtype,
eps: float,
n_extra: int,
seed: int,
):
"""
Test the batch-invariant fused residual-add + RMSNorm helper directly.
"""
device = torch.device(DEVICE_TYPE)
torch.manual_seed(42)
torch.manual_seed(seed)
x_single = torch.randn(1, hidden_size, dtype=dtype, device=device)
residual_single = torch.randn(1, hidden_size, dtype=dtype, device=device)
weight = torch.randn(hidden_size, dtype=dtype, device=device)
@@ -97,14 +103,14 @@ def test_fused_add_rms_norm_batch_invariant_residual_path(
x_batch = torch.cat(
[
x_single,
torch.randn(3, hidden_size, dtype=dtype, device=device),
torch.randn(n_extra, hidden_size, dtype=dtype, device=device),
],
dim=0,
)
residual_batch = torch.cat(
[
residual_single,
torch.randn(3, hidden_size, dtype=dtype, device=device),
torch.randn(n_extra, hidden_size, dtype=dtype, device=device),
],
dim=0,
)
@@ -168,6 +174,138 @@ def test_fused_add_rms_norm_batch_invariant_residual_path(
)
FP8_DTYPE = current_platform.fp8_dtype()
# The large launch (num_tokens=300 >= 256) drops an un-pinned kernel to block
# 256, while the small launch (255 rows) stays under the threshold and keeps the
# larger block (1024, or 512 for per-block quant). Under the pin the two launches
# use the same block, so the shared first 255 rows must match bit-for-bit; 255 is
# the most rows a single small launch can hold (< 256, and <= 256 for per-block).
_LARGE_TOKENS = 300
_SMALL_TOKENS = 255
def _assert_rows_bit_identical(small, large, msg):
if small.dtype == FP8_DTYPE:
assert torch.equal(small.view(torch.uint8), large.view(torch.uint8)), msg
else:
torch.testing.assert_close(small, large, rtol=0.0, atol=0.0, msg=msg)
@skip_if_not_cuda
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("seed", list(range(4)))
def test_rms_norm_batch_invariant_nonresidual_kernel(
hidden_size: int, dtype: torch.dtype, seed: int
):
"""C++ ``rms_norm`` (no residual) must be batch invariant across the block
threshold. Reached in compiled mode with ``ir_op_priority.rms_norm=["vllm_c"]``
(default priority is ``native``/inductor codegen when compiling).
"""
import vllm._custom_ops as ops
device = torch.device(DEVICE_TYPE)
torch.manual_seed(seed)
rows = torch.randn(_LARGE_TOKENS, hidden_size, dtype=dtype, device=device)
weight = torch.randn(hidden_size, dtype=dtype, device=device)
def rms_norm(x):
out = torch.empty_like(x)
ops.rms_norm(out, x, weight, 1e-6)
return out
large = rms_norm(rows.clone())
small = rms_norm(rows[:_SMALL_TOKENS].clone())
_assert_rows_bit_identical(
small,
large[:_SMALL_TOKENS],
"rms_norm output depends on num_tokens (block size)",
)
@skip_if_not_cuda
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("seed", list(range(4)))
@pytest.mark.parametrize("add_residual", [False, True])
def test_rms_norm_static_fp8_quant_batch_invariant(
hidden_size: int, dtype: torch.dtype, seed: int, add_residual: bool
):
"""C++ static per-tensor fp8-quant RMSNorm must be batch invariant across
the block threshold. Covers ``rms_norm_static_fp8_quant`` and, with
``add_residual``, ``fused_add_rms_norm_static_fp8_quant`` (the compiled fp8
path where ``RMSNormQuantFusionPass`` rewrites norm + quant into them).
"""
device = torch.device(DEVICE_TYPE)
torch.manual_seed(seed)
rows = torch.randn(_LARGE_TOKENS, hidden_size, dtype=dtype, device=device)
residual = (
torch.randn(_LARGE_TOKENS, hidden_size, dtype=dtype, device=device)
if add_residual
else None
)
weight = torch.randn(hidden_size, dtype=dtype, device=device)
quant_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
def quant(x, res):
out = torch.empty_like(x, dtype=FP8_DTYPE)
if add_residual:
torch.ops._C.fused_add_rms_norm_static_fp8_quant(
out, x, res, weight, quant_scale, 1e-6
)
else:
torch.ops._C.rms_norm_static_fp8_quant(out, x, weight, quant_scale, 1e-6)
return out
large = quant(rows.clone(), residual.clone() if residual is not None else None)
small = quant(
rows[:_SMALL_TOKENS].clone(),
residual[:_SMALL_TOKENS].clone() if residual is not None else None,
)
_assert_rows_bit_identical(
small,
large[:_SMALL_TOKENS],
"static-fp8-quant RMSNorm output depends on num_tokens (block size)",
)
@skip_if_not_cuda
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("seed", list(range(4)))
def test_rms_norm_per_block_quant_batch_invariant(
hidden_size: int, dtype: torch.dtype, seed: int
):
"""C++ ``rms_norm_per_block_quant`` must be batch invariant across the
block threshold (compiled fp8 block-quant path; block pinned to 512)."""
import vllm._custom_ops as ops
device = torch.device(DEVICE_TYPE)
torch.manual_seed(seed)
rows = torch.randn(_LARGE_TOKENS, hidden_size, dtype=dtype, device=device)
weight = torch.randn(hidden_size, dtype=dtype, device=device)
group_size = [1, 128]
def per_block_quant(x):
return ops.rms_norm_per_block_quant(x, weight, 1e-6, FP8_DTYPE, group_size)
out_large, scale_large = per_block_quant(rows.clone())
out_small, scale_small = per_block_quant(rows[:_SMALL_TOKENS].clone())
_assert_rows_bit_identical(
out_small,
out_large[:_SMALL_TOKENS],
"rms_norm_per_block_quant output depends on num_tokens (block size)",
)
torch.testing.assert_close(
scale_small,
scale_large[:_SMALL_TOKENS],
rtol=0.0,
atol=0.0,
msg="rms_norm_per_block_quant scales depend on num_tokens (block size)",
)
@skip_if_not_cuda
@pytest.mark.parametrize("batch_size", [1, 16, 128])
@pytest.mark.parametrize("seq_len", [1, 32, 512])