[vLLM IR] Add IR op testing and benchmarking infrastructure (#40167)
Signed-off-by: Yanan Cao <gmagogsfm@gmail.com> Co-authored-by: Theresa Shan <Theresa.Shan@amd.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Theresa Shan
Claude Opus 4.6
parent
6867bcd076
commit
fe5c115ee4
@@ -19,3 +19,18 @@ def rms_norm(
|
||||
if weight is not None:
|
||||
x = x.to(weight.dtype) * weight
|
||||
return x.to(orig_dtype)
|
||||
|
||||
|
||||
@rms_norm.register_input_generator
|
||||
def _rms_norm_input_generator(
|
||||
num_tokens: int, hidden_size: int, dtype: torch.dtype, epsilon: float = 1e-5
|
||||
) -> tuple:
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype)
|
||||
weight = torch.randn(hidden_size, dtype=dtype)
|
||||
return (x, weight, epsilon)
|
||||
|
||||
|
||||
# Reductions in rms_norm accumulate rounding error at large shapes
|
||||
# (e.g. 32768x16384), causing a few elements out of millions to exceed
|
||||
# the default float16 tolerance.
|
||||
rms_norm.override_tolerance(torch.float16, atol=1e-2, rtol=2e-3)
|
||||
|
||||
Reference in New Issue
Block a user