[Bugfix][Kernel] Fix batch invariance in RMSNorm kernels by pinning block size (#48391)
Signed-off-by: oops-oom <73481342@qq.com> Signed-off-by: oops-oom <liubin8905@vip.qq.com> Co-authored-by: oops-oom <73481342@qq.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
co-authored by
oops-oom
Claude Opus 4.8
Wentao Ye
Shengqi Chen
parent
94100b5915
commit
b6cbba8bc8
@@ -27,8 +27,10 @@ from vllm.platforms import current_platform
|
||||
"backend",
|
||||
BACKENDS,
|
||||
)
|
||||
@pytest.mark.parametrize("rms_norm_impl", ["default", "vllm_c"])
|
||||
def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
|
||||
backend,
|
||||
rms_norm_impl,
|
||||
):
|
||||
"""
|
||||
Ensures that the same request (the 'needle' prompt) yields identical output
|
||||
@@ -60,6 +62,16 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
|
||||
random.seed(seed)
|
||||
|
||||
attention_config = {"backend": backend}
|
||||
# Force the C++ RMSNorm implementation so we actually exercise the
|
||||
# num_tokens-dependent block-size branches.
|
||||
kernel_config = None
|
||||
if rms_norm_impl == "vllm_c":
|
||||
kernel_config = {
|
||||
"ir_op_priority": {
|
||||
"rms_norm": ["vllm_c"],
|
||||
"fused_add_rms_norm": ["vllm_c"],
|
||||
}
|
||||
}
|
||||
# Allow overrides from environment (useful for CI tuning)
|
||||
# "facebook/opt-125m" is too small, doesn't reliably test determinism
|
||||
model = TEST_MODEL
|
||||
@@ -96,6 +108,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
|
||||
gpu_memory_utilization=gpu_mem_util,
|
||||
max_model_len=max_model_len,
|
||||
attention_config=attention_config,
|
||||
kernel_config=kernel_config,
|
||||
)
|
||||
|
||||
# Baseline generation for the needle prompt alone.
|
||||
@@ -923,11 +936,15 @@ def LLM_with_max_seqs(
|
||||
gpu_memory_utilization: float,
|
||||
max_model_len: int,
|
||||
attention_config: dict | None = None,
|
||||
kernel_config: dict | None = None,
|
||||
) -> LLM:
|
||||
"""
|
||||
Helper to construct an LLM with a specific max_num_seqs (batch-size limit)
|
||||
using the high-level v1 LLM API, while constraining memory usage.
|
||||
"""
|
||||
extra_kwargs: dict = {}
|
||||
if kernel_config is not None:
|
||||
extra_kwargs["kernel_config"] = kernel_config
|
||||
return LLM(
|
||||
model=model,
|
||||
max_num_seqs=max_num_seqs,
|
||||
@@ -939,4 +956,5 @@ def LLM_with_max_seqs(
|
||||
attention_config=attention_config,
|
||||
# Enable for MOE models
|
||||
# enable_expert_parallel=True,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user