[Bugfix][Kernel] Fix batch invariance in RMSNorm kernels by pinning block size (#48391)
Signed-off-by: oops-oom <73481342@qq.com> Signed-off-by: oops-oom <liubin8905@vip.qq.com> Co-authored-by: oops-oom <73481342@qq.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
co-authored by
oops-oom
Claude Opus 4.8
Wentao Ye
Shengqi Chen
parent
94100b5915
commit
b6cbba8bc8
@@ -27,8 +27,10 @@ from vllm.platforms import current_platform
|
||||
"backend",
|
||||
BACKENDS,
|
||||
)
|
||||
@pytest.mark.parametrize("rms_norm_impl", ["default", "vllm_c"])
|
||||
def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
|
||||
backend,
|
||||
rms_norm_impl,
|
||||
):
|
||||
"""
|
||||
Ensures that the same request (the 'needle' prompt) yields identical output
|
||||
@@ -60,6 +62,16 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
|
||||
random.seed(seed)
|
||||
|
||||
attention_config = {"backend": backend}
|
||||
# Force the C++ RMSNorm implementation so we actually exercise the
|
||||
# num_tokens-dependent block-size branches.
|
||||
kernel_config = None
|
||||
if rms_norm_impl == "vllm_c":
|
||||
kernel_config = {
|
||||
"ir_op_priority": {
|
||||
"rms_norm": ["vllm_c"],
|
||||
"fused_add_rms_norm": ["vllm_c"],
|
||||
}
|
||||
}
|
||||
# Allow overrides from environment (useful for CI tuning)
|
||||
# "facebook/opt-125m" is too small, doesn't reliably test determinism
|
||||
model = TEST_MODEL
|
||||
@@ -96,6 +108,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
|
||||
gpu_memory_utilization=gpu_mem_util,
|
||||
max_model_len=max_model_len,
|
||||
attention_config=attention_config,
|
||||
kernel_config=kernel_config,
|
||||
)
|
||||
|
||||
# Baseline generation for the needle prompt alone.
|
||||
@@ -923,11 +936,15 @@ def LLM_with_max_seqs(
|
||||
gpu_memory_utilization: float,
|
||||
max_model_len: int,
|
||||
attention_config: dict | None = None,
|
||||
kernel_config: dict | None = None,
|
||||
) -> LLM:
|
||||
"""
|
||||
Helper to construct an LLM with a specific max_num_seqs (batch-size limit)
|
||||
using the high-level v1 LLM API, while constraining memory usage.
|
||||
"""
|
||||
extra_kwargs: dict = {}
|
||||
if kernel_config is not None:
|
||||
extra_kwargs["kernel_config"] = kernel_config
|
||||
return LLM(
|
||||
model=model,
|
||||
max_num_seqs=max_num_seqs,
|
||||
@@ -939,4 +956,5 @@ def LLM_with_max_seqs(
|
||||
attention_config=attention_config,
|
||||
# Enable for MOE models
|
||||
# enable_expert_parallel=True,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
@@ -28,16 +28,18 @@ def _rms_norm_reference(
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@pytest.mark.parametrize("batch_size", [1, 4, 16, 64])
|
||||
@pytest.mark.parametrize("batch_size", [1, 4, 64, 300])
|
||||
@pytest.mark.parametrize("hidden_size", [512, 2048, 4096, 8192])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("eps", [1e-6, 1e-5])
|
||||
@pytest.mark.parametrize("seed", list(range(4)))
|
||||
def test_rms_norm_batch_invariant_vs_reference(
|
||||
default_vllm_config,
|
||||
batch_size: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
eps: float,
|
||||
seed: int,
|
||||
):
|
||||
"""
|
||||
Compare batch-invariant Triton RMS norm against a PyTorch reference.
|
||||
@@ -48,7 +50,7 @@ def test_rms_norm_batch_invariant_vs_reference(
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
|
||||
# Create test input and weight
|
||||
torch.manual_seed(42)
|
||||
torch.manual_seed(seed)
|
||||
input_tensor = torch.randn(batch_size, hidden_size, dtype=dtype, device=device)
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
|
||||
@@ -71,7 +73,7 @@ def test_rms_norm_batch_invariant_vs_reference(
|
||||
atol=atol,
|
||||
msg=f"RMS norm mismatch for batch_size={batch_size}, "
|
||||
f"hidden_size={hidden_size}, "
|
||||
f"dtype={dtype}, eps={eps}",
|
||||
f"dtype={dtype}, eps={eps}, seed={seed}",
|
||||
)
|
||||
|
||||
|
||||
@@ -79,17 +81,21 @@ def test_rms_norm_batch_invariant_vs_reference(
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("eps", [1e-6])
|
||||
@pytest.mark.parametrize("n_extra", [3, 299])
|
||||
@pytest.mark.parametrize("seed", list(range(16)))
|
||||
def test_fused_add_rms_norm_batch_invariant_residual_path(
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
eps: float,
|
||||
n_extra: int,
|
||||
seed: int,
|
||||
):
|
||||
"""
|
||||
Test the batch-invariant fused residual-add + RMSNorm helper directly.
|
||||
"""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
|
||||
torch.manual_seed(42)
|
||||
torch.manual_seed(seed)
|
||||
x_single = torch.randn(1, hidden_size, dtype=dtype, device=device)
|
||||
residual_single = torch.randn(1, hidden_size, dtype=dtype, device=device)
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
@@ -97,14 +103,14 @@ def test_fused_add_rms_norm_batch_invariant_residual_path(
|
||||
x_batch = torch.cat(
|
||||
[
|
||||
x_single,
|
||||
torch.randn(3, hidden_size, dtype=dtype, device=device),
|
||||
torch.randn(n_extra, hidden_size, dtype=dtype, device=device),
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
residual_batch = torch.cat(
|
||||
[
|
||||
residual_single,
|
||||
torch.randn(3, hidden_size, dtype=dtype, device=device),
|
||||
torch.randn(n_extra, hidden_size, dtype=dtype, device=device),
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
@@ -168,6 +174,138 @@ def test_fused_add_rms_norm_batch_invariant_residual_path(
|
||||
)
|
||||
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
# The large launch (num_tokens=300 >= 256) drops an un-pinned kernel to block
|
||||
# 256, while the small launch (255 rows) stays under the threshold and keeps the
|
||||
# larger block (1024, or 512 for per-block quant). Under the pin the two launches
|
||||
# use the same block, so the shared first 255 rows must match bit-for-bit; 255 is
|
||||
# the most rows a single small launch can hold (< 256, and <= 256 for per-block).
|
||||
_LARGE_TOKENS = 300
|
||||
_SMALL_TOKENS = 255
|
||||
|
||||
|
||||
def _assert_rows_bit_identical(small, large, msg):
|
||||
if small.dtype == FP8_DTYPE:
|
||||
assert torch.equal(small.view(torch.uint8), large.view(torch.uint8)), msg
|
||||
else:
|
||||
torch.testing.assert_close(small, large, rtol=0.0, atol=0.0, msg=msg)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("seed", list(range(4)))
|
||||
def test_rms_norm_batch_invariant_nonresidual_kernel(
|
||||
hidden_size: int, dtype: torch.dtype, seed: int
|
||||
):
|
||||
"""C++ ``rms_norm`` (no residual) must be batch invariant across the block
|
||||
threshold. Reached in compiled mode with ``ir_op_priority.rms_norm=["vllm_c"]``
|
||||
(default priority is ``native``/inductor codegen when compiling).
|
||||
"""
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
torch.manual_seed(seed)
|
||||
rows = torch.randn(_LARGE_TOKENS, hidden_size, dtype=dtype, device=device)
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
|
||||
def rms_norm(x):
|
||||
out = torch.empty_like(x)
|
||||
ops.rms_norm(out, x, weight, 1e-6)
|
||||
return out
|
||||
|
||||
large = rms_norm(rows.clone())
|
||||
small = rms_norm(rows[:_SMALL_TOKENS].clone())
|
||||
_assert_rows_bit_identical(
|
||||
small,
|
||||
large[:_SMALL_TOKENS],
|
||||
"rms_norm output depends on num_tokens (block size)",
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("seed", list(range(4)))
|
||||
@pytest.mark.parametrize("add_residual", [False, True])
|
||||
def test_rms_norm_static_fp8_quant_batch_invariant(
|
||||
hidden_size: int, dtype: torch.dtype, seed: int, add_residual: bool
|
||||
):
|
||||
"""C++ static per-tensor fp8-quant RMSNorm must be batch invariant across
|
||||
the block threshold. Covers ``rms_norm_static_fp8_quant`` and, with
|
||||
``add_residual``, ``fused_add_rms_norm_static_fp8_quant`` (the compiled fp8
|
||||
path where ``RMSNormQuantFusionPass`` rewrites norm + quant into them).
|
||||
"""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
torch.manual_seed(seed)
|
||||
rows = torch.randn(_LARGE_TOKENS, hidden_size, dtype=dtype, device=device)
|
||||
residual = (
|
||||
torch.randn(_LARGE_TOKENS, hidden_size, dtype=dtype, device=device)
|
||||
if add_residual
|
||||
else None
|
||||
)
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
quant_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
def quant(x, res):
|
||||
out = torch.empty_like(x, dtype=FP8_DTYPE)
|
||||
if add_residual:
|
||||
torch.ops._C.fused_add_rms_norm_static_fp8_quant(
|
||||
out, x, res, weight, quant_scale, 1e-6
|
||||
)
|
||||
else:
|
||||
torch.ops._C.rms_norm_static_fp8_quant(out, x, weight, quant_scale, 1e-6)
|
||||
return out
|
||||
|
||||
large = quant(rows.clone(), residual.clone() if residual is not None else None)
|
||||
small = quant(
|
||||
rows[:_SMALL_TOKENS].clone(),
|
||||
residual[:_SMALL_TOKENS].clone() if residual is not None else None,
|
||||
)
|
||||
_assert_rows_bit_identical(
|
||||
small,
|
||||
large[:_SMALL_TOKENS],
|
||||
"static-fp8-quant RMSNorm output depends on num_tokens (block size)",
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("seed", list(range(4)))
|
||||
def test_rms_norm_per_block_quant_batch_invariant(
|
||||
hidden_size: int, dtype: torch.dtype, seed: int
|
||||
):
|
||||
"""C++ ``rms_norm_per_block_quant`` must be batch invariant across the
|
||||
block threshold (compiled fp8 block-quant path; block pinned to 512)."""
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
torch.manual_seed(seed)
|
||||
rows = torch.randn(_LARGE_TOKENS, hidden_size, dtype=dtype, device=device)
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
group_size = [1, 128]
|
||||
|
||||
def per_block_quant(x):
|
||||
return ops.rms_norm_per_block_quant(x, weight, 1e-6, FP8_DTYPE, group_size)
|
||||
|
||||
out_large, scale_large = per_block_quant(rows.clone())
|
||||
out_small, scale_small = per_block_quant(rows[:_SMALL_TOKENS].clone())
|
||||
_assert_rows_bit_identical(
|
||||
out_small,
|
||||
out_large[:_SMALL_TOKENS],
|
||||
"rms_norm_per_block_quant output depends on num_tokens (block size)",
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
scale_small,
|
||||
scale_large[:_SMALL_TOKENS],
|
||||
rtol=0.0,
|
||||
atol=0.0,
|
||||
msg="rms_norm_per_block_quant scales depend on num_tokens (block size)",
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@pytest.mark.parametrize("batch_size", [1, 16, 128])
|
||||
@pytest.mark.parametrize("seq_len", [1, 32, 512])
|
||||
|
||||
Reference in New Issue
Block a user