[Kernel][Helion] Add numerics checks to benchmark script (#48968)

Signed-off-by: Shangdi Yu <shangdiy@meta.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Shangdi Yu
2026-07-31 20:16:02 -04:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 4.8
parent e3be89673d
commit b40d859c7b
2 changed files with 339 additions and 10 deletions
+249 -10
View File
@@ -5,8 +5,10 @@
Benchmark a registered Helion kernel against a baseline. Benchmark a registered Helion kernel against a baseline.
For each input case produced by the kernel's registered input generator, this For each input case produced by the kernel's registered input generator, this
measures the latency of the Helion kernel and a chosen baseline, then reports checks the Helion kernel's numerics once against its eager reference, measures
the speedup. its latency against a chosen performance baseline, then reports the speedup.
Use ``--numerics-with-perf-baseline`` to check numerics against the performance
baseline instead.
Two baselines are supported (``--baseline``): Two baselines are supported (``--baseline``):
@@ -30,9 +32,17 @@ Usage:
python scripts/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\ python scripts/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
--baseline cuda --baseline cuda
# Check numerics against the performance baseline instead of eager
python scripts/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
--baseline cuda --numerics-with-perf-baseline
# Disable CUDA graph capture and save results # Disable CUDA graph capture and save results
python scripts/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\ python scripts/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
--no-cudagraph --output results.json --no-cudagraph --output results.json
# Only verify numerics, skipping the timing runs
python scripts/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
--numerics-only
""" """
import argparse import argparse
@@ -43,15 +53,19 @@ import statistics
import sys import sys
from collections.abc import Callable from collections.abc import Callable
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from typing import Any
import torch import torch
from torch.utils._pytree import tree_flatten
from vllm.triton_utils import triton from vllm.triton_utils import triton
try: try:
from helion.autotuner.accuracy import assert_close as helion_assert_close
from helion.autotuner.accuracy import is_fp8_dtype
from vllm.benchmarks.lib.utils import default_vllm_config from vllm.benchmarks.lib.utils import default_vllm_config
from vllm.kernels.helion import get_kernel_by_name, get_registered_kernels from vllm.kernels.helion import get_kernel_by_name, get_registered_kernels
from vllm.kernels.helion.ops import import_all_kernels
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion from vllm.utils.import_utils import has_helion
except ImportError as e: except ImportError as e:
@@ -59,6 +73,27 @@ except ImportError as e:
print("Please ensure vLLM is installed and in your Python path") print("Please ensure vLLM is installed and in your Python path")
sys.exit(1) sys.exit(1)
def import_all_kernels() -> None:
"""Trigger Helion op registration, tolerating cross-version name drift.
Current vLLM registers every Helion kernel as a side effect of importing
``vllm.kernels.helion.ops``; some builds instead expose an explicit importer
whose name has drifted (``import_all_kernels`` / ``import_all_ops``). Call
whichever exists; if none does, importing the module already registered
them.
"""
try:
import vllm.kernels.helion.ops as ops
except ImportError:
return
for fn_name in ("import_all_kernels", "import_all_ops"):
fn = getattr(ops, fn_name, None)
if callable(fn):
fn()
return
logger = init_logger("vllm.scripts.benchmark_helion_kernels") logger = init_logger("vllm.scripts.benchmark_helion_kernels")
@@ -121,6 +156,21 @@ def print_table(rows: list[Row]) -> None:
print(fmt(row)) print(fmt(row))
def log_versions() -> None:
"""Log torch/helion/triton versions at the head of the output."""
from importlib.metadata import PackageNotFoundError, version
def pkg_version(name: str) -> str:
try:
return version(name)
except PackageNotFoundError:
return "not installed"
logger.info("torch: %s", torch.__version__)
logger.info("helion: %s", pkg_version("helion"))
logger.info("triton: %s", getattr(triton, "__version__", pkg_version("triton")))
def list_kernels() -> None: def list_kernels() -> None:
kernels = get_registered_kernels() kernels = get_registered_kernels()
@@ -176,12 +226,8 @@ def make_cuda_baseline(kernel_name: str) -> Callable:
return cuda_op return cuda_op
def make_autotune_baseline(kernel_name: str) -> Callable: def make_eager_baseline(kernel_name: str) -> Callable:
"""Return the kernel's autotune baseline wrapped in ``torch.compile``. """Return the kernel's registered native-torch reference."""
The baseline is the native-torch reference the kernel is tuned against,
registered via ``helion_settings.autotune_baseline_fn``.
"""
wrapper = get_kernel_by_name(kernel_name) wrapper = get_kernel_by_name(kernel_name)
settings = wrapper.helion_settings settings = wrapper.helion_settings
baseline_fn = getattr(settings, "autotune_baseline_fn", None) baseline_fn = getattr(settings, "autotune_baseline_fn", None)
@@ -195,6 +241,13 @@ def make_autotune_baseline(kernel_name: str) -> Callable:
) )
sys.exit(1) sys.exit(1)
return baseline_fn
def make_autotune_baseline(kernel_name: str) -> Callable:
"""Return the kernel's autotune baseline wrapped in ``torch.compile``."""
baseline_fn = make_eager_baseline(kernel_name)
return torch.compile( return torch.compile(
baseline_fn, baseline_fn,
fullgraph=True, fullgraph=True,
@@ -204,6 +257,25 @@ def make_autotune_baseline(kernel_name: str) -> Callable:
) )
def make_correctness_baseline(
kernel_name: str,
timed_baseline_fn: Callable,
numerics_with_perf_baseline: bool,
) -> Callable:
"""Choose the numerical reference independently from the timed baseline."""
if not numerics_with_perf_baseline:
logger.info(
"Using the eager reference for '%s' correctness",
kernel_name,
)
return make_eager_baseline(kernel_name)
logger.info(
"Using the selected performance baseline for '%s' correctness",
kernel_name,
)
return timed_baseline_fn
def cleanup_gpu_resources() -> None: def cleanup_gpu_resources() -> None:
try: try:
torch.accelerator.empty_cache() torch.accelerator.empty_cache()
@@ -227,6 +299,118 @@ def _reduce(times: list[float], return_mode: str) -> float:
return _REDUCERS[return_mode](times) return _REDUCERS[return_mode](times)
def _assert_close(actual: object, expected: object, atol: float, rtol: float) -> None:
"""Compare pytrees, allowing the one-ULP FP8 variance used by kernel tests."""
actual_flat, actual_spec = tree_flatten(actual)
expected_flat, expected_spec = tree_flatten(expected)
if actual_spec != expected_spec:
raise AssertionError(
f"Output structure mismatch: {actual_spec} != {expected_spec}"
)
for actual_leaf, expected_leaf in zip(actual_flat, expected_flat, strict=True):
is_fp8 = isinstance(actual_leaf, torch.Tensor) and is_fp8_dtype(
actual_leaf.dtype
)
helion_assert_close(
actual_leaf,
expected_leaf,
atol=1 if is_fp8 else atol,
rtol=0 if is_fp8 else rtol,
)
def check_correctness(
kernel: Any,
baseline_fn: Callable,
inputs: tuple[Any, ...],
case: str,
) -> None:
"""Run one numerical comparison on copies separate from benchmark inputs."""
kernel_inputs = copy.deepcopy(inputs)
baseline_inputs = copy.deepcopy(inputs)
kernel_output = kernel(*kernel_inputs)
baseline_output = baseline_fn(*baseline_inputs)
settings = kernel.helion_settings
try:
custom_check = getattr(settings, "autotune_baseline_accuracy_check_fn", None)
if custom_check is not None:
custom_check(kernel_output, baseline_output)
custom_check(kernel_inputs, baseline_inputs)
return
configured_atol = getattr(settings, "autotune_baseline_atol", None)
configured_rtol = getattr(settings, "autotune_baseline_rtol", None)
atol = 1e-2 if configured_atol is None else configured_atol
rtol = 1e-2 if configured_rtol is None else configured_rtol
_assert_close(
kernel_output,
baseline_output,
atol=atol,
rtol=rtol,
)
_assert_close(
kernel_inputs,
baseline_inputs,
atol=atol,
rtol=rtol,
)
except AssertionError as e:
raise AssertionError(f"Numerics check failed for case {case}:\n{e}") from e
@dataclass
class CorrectnessResult:
"""Outcome of the numerics check for a single shape case."""
case: str
passed: bool
error: str | None = None
def check_kernel_correctness(
kernel: Any,
baseline_fn: Callable,
inputs_dict: dict[Any, tuple[Any, ...]] | None = None,
) -> list[CorrectnessResult]:
"""Run the per-shape numerics check for a kernel, continuing past failures.
Runs the same comparison as ``check_correctness`` for every shape case
produced by the kernel's input generator, but records the outcome per case
instead of raising on the first mismatch. This lets callers (e.g. a CI gate)
report every failing shape in one pass rather than aborting early.
Args:
kernel: The Helion kernel wrapper to check.
baseline_fn: Reference callable sharing the kernel's argument interface.
inputs_dict: Optional mapping of case key to input tuple. Defaults to
``kernel.get_inputs()``.
Returns:
One ``CorrectnessResult`` per shape case, in iteration order. A case that
raises (compile/run error or numerics mismatch) is marked
``passed=False`` with the exception text in ``error``; iteration
continues regardless. An empty input mapping returns an empty list, which
the CLI reports as a skipped check.
"""
if inputs_dict is None:
inputs_dict = kernel.get_inputs()
results: list[CorrectnessResult] = []
for key, inputs in inputs_dict.items():
case = str(key)
try:
check_correctness(kernel, baseline_fn, inputs, case)
except Exception as e: # noqa: BLE001 - any failure is recorded, not fatal
results.append(CorrectnessResult(case=case, passed=False, error=str(e)))
else:
results.append(CorrectnessResult(case=case, passed=True))
cleanup_gpu_resources()
return results
def do_bench_cudagraph_l2_clear( def do_bench_cudagraph_l2_clear(
fn: Callable, rep: int = 100, return_mode: str = "mean" fn: Callable, rep: int = 100, return_mode: str = "mean"
) -> float: ) -> float:
@@ -298,6 +482,7 @@ def do_bench_cudagraph_l2_clear(
def benchmark( def benchmark(
kernel_name: str, kernel_name: str,
baseline_fn: Callable, baseline_fn: Callable,
correctness_fn: Callable,
repeat: int, repeat: int,
cudagraph: bool, cudagraph: bool,
return_mode: str, return_mode: str,
@@ -313,6 +498,9 @@ def benchmark(
for key, inputs in inputs_dict.items(): for key, inputs in inputs_dict.items():
logger.info("Benchmarking case %s", key) logger.info("Benchmarking case %s", key)
check_correctness(kernel, correctness_fn, inputs, str(key))
logger.info("Numerics check passed for case %s", key)
# Kernels may mutate their inputs in place; give each side its own copy. # Kernels may mutate their inputs in place; give each side its own copy.
kernel_inputs = copy.deepcopy(inputs) kernel_inputs = copy.deepcopy(inputs)
baseline_inputs = copy.deepcopy(inputs) baseline_inputs = copy.deepcopy(inputs)
@@ -374,7 +562,7 @@ def main() -> None:
choices=["cuda", "autotune"], choices=["cuda", "autotune"],
default="autotune", default="autotune",
help=( help=(
"Baseline to compare against: 'autotune' uses the kernel's " "Performance baseline: 'autotune' uses the kernel's "
"autotune_baseline_fn under torch.compile; 'cuda' uses the mapped " "autotune_baseline_fn under torch.compile; 'cuda' uses the mapped "
"torch.ops._C op (default: autotune)" "torch.ops._C op (default: autotune)"
), ),
@@ -390,9 +578,24 @@ def main() -> None:
type=str, type=str,
help="Path to save benchmark results as JSON (default: log only)", help="Path to save benchmark results as JSON (default: log only)",
) )
parser.add_argument(
"--numerics-only",
action="store_true",
help="Only run the per-case numerics check; skip timing and reporting",
)
parser.add_argument(
"--numerics-with-perf-baseline",
action="store_true",
help=(
"Compare numerics against the selected performance baseline instead "
"of the eager reference"
),
)
args = parser.parse_args() args = parser.parse_args()
log_versions()
import_all_kernels() import_all_kernels()
if args.list: if args.list:
@@ -425,10 +628,43 @@ def main() -> None:
baseline_fn = make_cuda_baseline(args.kernel) baseline_fn = make_cuda_baseline(args.kernel)
else: else:
baseline_fn = make_autotune_baseline(args.kernel) baseline_fn = make_autotune_baseline(args.kernel)
correctness_fn = make_correctness_baseline(
args.kernel,
baseline_fn,
args.numerics_with_perf_baseline,
)
if args.numerics_only:
results = check_kernel_correctness(wrapper, correctness_fn)
if not results:
logger.warning(
"No input cases generated for '%s'; skipping numerics check",
args.kernel,
)
return
for r in results:
if r.passed:
logger.info("Numerics check passed for case %s", r.case)
else:
logger.error(
"Numerics check FAILED for case %s: %s", r.case, r.error
)
failed = [r for r in results if not r.passed]
if failed:
logger.error(
"%d/%d case(s) failed numerics for '%s'",
len(failed),
len(results),
args.kernel,
)
sys.exit(1)
logger.info("Numerics check passed for all cases of '%s'", args.kernel)
return
rows = benchmark( rows = benchmark(
args.kernel, args.kernel,
baseline_fn, baseline_fn,
correctness_fn,
args.repeat, args.repeat,
args.cudagraph, args.cudagraph,
args.return_mode, args.return_mode,
@@ -442,6 +678,9 @@ def main() -> None:
{ {
"kernel": args.kernel, "kernel": args.kernel,
"baseline": args.baseline, "baseline": args.baseline,
"numerics_baseline": (
args.baseline if args.numerics_with_perf_baseline else "eager"
),
"cudagraph": args.cudagraph, "cudagraph": args.cudagraph,
"repeat": args.repeat, "repeat": args.repeat,
"return_mode": args.return_mode, "return_mode": args.return_mode,
@@ -0,0 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.utils.import_utils import has_helion
if not has_helion():
pytest.skip("Helion is not installed", allow_module_level=True)
from scripts import benchmark_helion_kernels
check_correctness = benchmark_helion_kernels.check_correctness
class _FakeKernel:
helion_settings = None
def __init__(self, offset: float):
self.calls = 0
self.offset = offset
def __call__(self, output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
self.calls += 1
output.copy_(input + self.offset)
return input * 2
def test_check_correctness_runs_once_without_mutating_benchmark_inputs():
kernel = _FakeKernel(offset=1)
baseline_calls = 0
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
nonlocal baseline_calls
baseline_calls += 1
output.copy_(input + 1)
return input * 2
output = torch.zeros(4)
inputs = (output, torch.arange(4))
check_correctness(kernel, baseline, inputs, "matching")
assert kernel.calls == 1
assert baseline_calls == 1
torch.testing.assert_close(output, torch.zeros(4))
def test_check_correctness_reports_mutated_input_mismatch():
kernel = _FakeKernel(offset=2)
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
output.copy_(input + 1)
return input * 2
inputs = (torch.zeros(4), torch.arange(4))
with pytest.raises(AssertionError, match="Numerics check failed for case bad"):
check_correctness(kernel, baseline, inputs, "bad")
def test_check_correctness_reports_return_value_mismatch():
kernel = _FakeKernel(offset=1)
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
output.copy_(input + 1)
return input * 3
inputs = (torch.zeros(4), torch.arange(4))
with pytest.raises(AssertionError, match="Numerics check failed for case bad"):
check_correctness(kernel, baseline, inputs, "bad")
def test_log_versions(monkeypatch):
messages = []
def record(message: str, *args):
messages.append(message % args)
monkeypatch.setattr(benchmark_helion_kernels.logger, "info", record)
benchmark_helion_kernels.log_versions()
assert [message.split(":", 1)[0] for message in messages] == [
"torch",
"helion",
"triton",
]