[Refactor] Move auxiliary files out of the repository root (#58572)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Misha Goin
2026-09-24 12:38:56 -07:00
committed by GitHub
parent cd06e81c41
commit 58aa2f1ef1
35 changed files with 134 additions and 130 deletions
+438
View File
@@ -0,0 +1,438 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Autotune registered Helion kernels for optimal configurations.
Usage:
# Autotune all registered kernels
python tools/autotune_helion_kernels.py
# Autotune specific kernel
python tools/autotune_helion_kernels.py --kernels silu_mul_fp8
# Autotune multiple kernels
python tools/autotune_helion_kernels.py --kernels silu_mul_fp8 rms_norm_fp8
# Force re-autotuning
python tools/autotune_helion_kernels.py --force
# List available kernels
python tools/autotune_helion_kernels.py --list
"""
import argparse
import sys
import time
from dataclasses import dataclass
import torch
from torch._subclasses.fake_tensor import FakeTensorMode
try:
import helion
from vllm.kernels.helion import (
ConfigManager,
get_kernel_by_name,
get_registered_kernels,
)
from vllm.kernels.helion.ops import import_all_kernels
from vllm.kernels.helion.utils import get_canonical_gpu_name
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
except ImportError as e:
print(f"Error importing vLLM: {e}")
print("Please ensure vLLM is installed and in your Python path")
sys.exit(1)
logger = init_logger("vllm.scripts.autotune_helion_kernels")
@dataclass
class AutotuneResult:
status: str # "success" | "partial" | "error" | "skipped"
successful: int
failed: int
configs: dict[str, "helion.Config"]
message: str = ""
def list_kernels() -> None:
kernels = get_registered_kernels()
if not kernels:
print("No Helion kernels found in registry.")
return
print("Available Helion kernels:")
print("=" * 50)
for name in sorted(kernels.keys()):
print(f" {name}")
print(f"\nTotal: {len(kernels)} kernels")
def check_requirements() -> bool:
if not torch.cuda.is_available():
logger.error("CUDA is not available. Helion autotuning requires GPU.")
return False
if not has_helion():
logger.error("Helion is not installed. Please install Helion package.")
return False
return True
def autotune_kernel(
kernel_name: str,
platform: str,
config_manager: ConfigManager,
force: bool = False,
autotune_effort: str = "quick",
) -> AutotuneResult:
logger.debug(
"Starting autotune for kernel '%s' with effort='%s'",
kernel_name,
autotune_effort,
)
kernel_wrapper = get_kernel_by_name(kernel_name)
if kernel_wrapper is None:
error_msg = f"Kernel '{kernel_name}' not found in registry"
logger.error(error_msg)
return AutotuneResult(
status="error",
message=error_msg,
successful=0,
failed=0,
configs={},
)
try:
with FakeTensorMode():
all_config_keys = list(kernel_wrapper.get_inputs().keys())
except NotImplementedError:
error_msg = f"Kernel '{kernel_name}' has no input generator registered"
logger.error(error_msg)
return AutotuneResult(
status="error",
message=error_msg,
successful=0,
failed=0,
configs={},
)
try:
logger.info(
"Autotuning kernel '%s' for platform '%s' with %d configs",
kernel_name,
platform,
len(all_config_keys),
)
if not force:
existing_configs = config_manager.get_platform_configs(
kernel_name, platform
)
keys_to_autotune = []
for config_key in all_config_keys:
if config_key in existing_configs:
logger.debug(
"Config '%s' already exists for platform '%s', skipping",
config_key,
platform,
)
else:
keys_to_autotune.append(config_key)
else:
logger.debug("Force mode enabled, will re-autotune all configs")
keys_to_autotune = all_config_keys
if not keys_to_autotune:
logger.info(
"All configs already exist for kernel '%s' on platform '%s'. "
"Use --force to re-autotune.",
kernel_name,
platform,
)
return AutotuneResult(
status="skipped",
message="All configs already exist",
successful=0,
failed=0,
configs={},
)
inputs_dict = kernel_wrapper.get_inputs()
configs_to_autotune = {k: inputs_dict[k] for k in keys_to_autotune}
total_start_time = time.time()
autotuned_configs = {}
failed_configs = []
for config_key, inputs in configs_to_autotune.items():
logger.info("Autotuning config: %s", config_key)
logger.debug(
"Input shapes: %s",
[getattr(inp, "shape", type(inp).__name__) for inp in inputs],
)
try:
config_start_time = time.time()
config = kernel_wrapper.run_autotune(inputs, autotune_effort)
config_duration = time.time() - config_start_time
# Save immediately for checkpointing
config_manager.save_configs(kernel_name, platform, {config_key: config})
autotuned_configs[config_key] = config
logger.debug("Config details: %s", config)
logger.info(
"✓ Autotuned and saved config '%s' (%.2fs)",
config_key,
config_duration,
)
except (RuntimeError, ValueError, OSError) as e:
logger.exception(
"Failed to autotune config '%s': %s",
config_key,
e,
)
failed_configs.append(config_key)
total_duration = time.time() - total_start_time
successful = len(autotuned_configs)
failed = len(failed_configs)
logger.info(
"Completed autotuning for kernel '%s': %d successful, %d failed (%.2fs)",
kernel_name,
successful,
failed,
total_duration,
)
status = "success" if failed == 0 else "partial"
return AutotuneResult(
status=status,
successful=successful,
failed=failed,
configs=autotuned_configs,
)
except (KeyError, RuntimeError, ValueError, OSError) as e:
error_msg = f"Unexpected error: {e}"
logger.exception("Failed to autotune kernel '%s': %s", kernel_name, e)
return AutotuneResult(
status="error",
message=error_msg,
successful=0,
failed=0,
configs={},
)
def summarize_results(results: dict[str, AutotuneResult]) -> bool:
logger.info("=" * 50)
logger.info("Autotuning Results Summary")
logger.info("=" * 50)
total_successful = 0
total_failed = 0
success_kernels = []
partial_kernels = []
error_kernels = []
skipped_kernels = []
for kernel_name, result in results.items():
total_successful += result.successful
total_failed += result.failed
if result.status == "success":
success_kernels.append(f"{kernel_name} ({result.successful} configs)")
logger.info("✓ %s: %d configs successful", kernel_name, result.successful)
elif result.status == "partial":
partial_kernels.append(
f"{kernel_name} ({result.successful} ok, {result.failed} failed)"
)
logger.warning(
"⚠ %s: %d successful, %d failed",
kernel_name,
result.successful,
result.failed,
)
elif result.status == "error":
error_kernels.append(f"{kernel_name}: {result.message or 'Unknown error'}")
logger.error("✗ %s: %s", kernel_name, result.message or "Unknown error")
elif result.status == "skipped":
skipped_kernels.append(f"{kernel_name}: {result.message or 'Skipped'}")
logger.info("- %s: %s", kernel_name, result.message or "Skipped")
logger.info("=" * 50)
logger.info(
"Summary: %d total configs (%d successful, %d failed)",
total_successful + total_failed,
total_successful,
total_failed,
)
logger.info(
"Kernels: %d success, %d partial, %d error, %d skipped",
len(success_kernels),
len(partial_kernels),
len(error_kernels),
len(skipped_kernels),
)
has_failures = bool(error_kernels or partial_kernels)
if not has_failures:
if total_successful > 0:
logger.info("All configs autotuned successfully!")
else:
logger.info("No new configs were generated (all may already exist)")
return not has_failures
def get_kernels_to_autotune(requested_kernels: list[str] | None) -> list[str]:
all_kernels = get_registered_kernels()
if not all_kernels:
logger.error("No Helion kernels found in registry")
sys.exit(1)
if not requested_kernels:
return list(all_kernels.keys())
if len(requested_kernels) != len(set(requested_kernels)):
duplicates = [
k for k in set(requested_kernels) if requested_kernels.count(k) > 1
]
logger.error("Duplicate kernel names in --kernels flag: %s", duplicates)
sys.exit(1)
kernels_to_autotune = []
missing_kernels = []
for kernel_name in requested_kernels:
if kernel_name in all_kernels:
kernels_to_autotune.append(kernel_name)
else:
missing_kernels.append(kernel_name)
if missing_kernels:
logger.error("Kernel(s) not found: %s", missing_kernels)
logger.error("Available kernels: %s", list(all_kernels.keys()))
sys.exit(1)
return kernels_to_autotune
def main():
parser = argparse.ArgumentParser(
description="Autotune Helion kernels",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__.split("Usage:")[1] if "Usage:" in __doc__ else "",
)
parser.add_argument(
"--kernels",
nargs="+",
help="Kernel(s) to autotune (default: all kernels)",
)
parser.add_argument(
"--config-dir",
type=str,
help="Config directory for config files (default: vLLM helion configs dir)",
)
parser.add_argument(
"--list",
action="store_true",
help="List available Helion kernels and exit",
)
parser.add_argument(
"--force",
action="store_true",
help=(
"Force re-autotuning even if configs already exist for the "
"platform and config keys"
),
)
parser.add_argument(
"--autotune-effort",
type=str,
default="quick",
help=(
"Helion autotune effort level: 'quick' (smaller search) or "
"'full' (full search budget) (default: quick)"
),
)
parser.add_argument(
"--verbose",
action="store_true",
help="Enable verbose logging",
)
args = parser.parse_args()
# import all helion kernels
import_all_kernels()
import logging
if args.verbose:
logging.getLogger("vllm").setLevel(logging.DEBUG)
logger.debug("Verbose mode enabled")
logger.debug("Arguments: %s", vars(args))
else:
logging.getLogger("vllm").setLevel(logging.INFO)
if args.list:
list_kernels()
return
if not check_requirements():
sys.exit(1)
platform = get_canonical_gpu_name()
logger.info("Detected GPU platform: %s", platform)
config_manager = (
ConfigManager(args.config_dir) if args.config_dir else ConfigManager()
)
try:
config_manager.ensure_base_dir_writable()
except OSError as e:
logger.error("Failed to access config directory: %s", e)
sys.exit(1)
kernels_to_autotune = get_kernels_to_autotune(args.kernels)
logger.info(
"Will autotune %d kernel(s) for platform '%s': %s",
len(kernels_to_autotune),
platform,
kernels_to_autotune,
)
results = {}
for kernel_name in kernels_to_autotune:
result = autotune_kernel(
kernel_name, platform, config_manager, args.force, args.autotune_effort
)
results[kernel_name] = result
success = summarize_results(results)
sys.exit(0 if success else 1)
if __name__ == "__main__":
main()
+696
View File
@@ -0,0 +1,696 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark a registered Helion kernel against a baseline.
For each input case produced by the kernel's registered input generator, this
checks the Helion kernel's numerics once against its eager reference, measures
its latency against a chosen performance baseline, then reports the speedup.
Use ``--numerics-with-perf-baseline`` to check numerics against the performance
baseline instead.
Two baselines are supported (``--baseline``):
- ``autotune`` (default): the kernel's autotuning baseline
(``helion_settings.autotune_baseline_fn``), wrapped in ``torch.compile``
(inductor). This is the native-torch reference used by kernel autotuning
and correctness unit tests.
- ``cuda``: the corresponding hand-written CUDA op (``torch.ops._C.*``). The
mapping from Helion kernel name to CUDA op lives in ``CUDA_BASELINE_OPS``
below. Every Helion kernel shares the same argument interface as its CUDA
counterpart, so inputs are forwarded verbatim.
Usage:
# List available kernels
python tools/benchmark_helion_kernels.py --list
# Benchmark a kernel against the autotune baseline (default)
python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant
# Benchmark against the CUDA baseline
python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
--baseline cuda
# Check numerics against the performance baseline instead of eager
python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
--baseline cuda --numerics-with-perf-baseline
# Disable CUDA graph capture and save results
python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
--no-cudagraph --output results.json
# Only verify numerics, skipping the timing runs
python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
--numerics-only
"""
import argparse
import copy
import gc
import json
import statistics
import sys
from collections.abc import Callable
from dataclasses import asdict, dataclass
from typing import Any
import torch
from torch.utils._pytree import tree_flatten
from vllm.triton_utils import triton
try:
from helion.autotuner.accuracy import assert_close as helion_assert_close
from helion.autotuner.accuracy import is_fp8_dtype
from vllm.benchmarks.lib.utils import default_vllm_config
from vllm.kernels.helion import get_kernel_by_name, get_registered_kernels
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
except ImportError as e:
print(f"Error importing vLLM: {e}")
print("Please ensure vLLM is installed and in your Python path")
sys.exit(1)
def import_all_kernels() -> None:
"""Trigger Helion op registration, tolerating cross-version name drift.
Current vLLM registers every Helion kernel as a side effect of importing
``vllm.kernels.helion.ops``; some builds instead expose an explicit importer
whose name has drifted (``import_all_kernels`` / ``import_all_ops``). Call
whichever exists; if none does, importing the module already registered
them.
"""
try:
import vllm.kernels.helion.ops as ops
except ImportError:
return
for fn_name in ("import_all_kernels", "import_all_ops"):
fn = getattr(ops, fn_name, None)
if callable(fn):
fn()
return
logger = init_logger("vllm.scripts.benchmark_helion_kernels")
# Maps a Helion kernel name to the CUDA op (attribute on ``torch.ops._C``) that
# implements the same operation. Helion kernels share the CUDA op's argument
# interface, so the kernel's input tuple is forwarded verbatim. Add an entry
# here when introducing a new kernel whose baseline should be the CUDA op.
CUDA_BASELINE_OPS: dict[str, str] = {
"dynamic_per_token_scaled_fp8_quant": "dynamic_per_token_scaled_fp8_quant",
"fused_qk_norm_rope": "fused_qk_norm_rope",
"per_token_group_fp8_quant": "per_token_group_fp8_quant",
"rms_norm_dynamic_per_token_quant": "rms_norm_dynamic_per_token_quant",
"rms_norm_per_block_quant": "rms_norm_per_block_quant",
"silu_and_mul_per_block_quant": "silu_and_mul_per_block_quant",
"scaled_mm": "cutlass_scaled_mm",
}
# torch.compile options for the torch baseline, mirroring how these kernels are
# compiled inside vLLM.
_TORCH_COMPILE_OPTIONS: dict[str, bool] = {
"enable_auto_functionalized_v2": False,
"size_asserts": False,
"alignment_asserts": False,
"scalar_asserts": False,
"combo_kernels": True,
"benchmark_combo_kernel": True,
}
@dataclass
class Row:
case: str
baseline_ms: float
kernel_ms: float
speedup_x: float
def print_table(rows: list[Row]) -> None:
headers = ["case", "baseline_ms", "kernel_ms", "speedup(x)"]
data = [
[
r.case,
f"{r.baseline_ms:.3f}",
f"{r.kernel_ms:.3f}",
f"{r.speedup_x:.3f}",
]
for r in rows
]
cols = list(zip(*([headers] + data)))
widths = [max(len(cell) for cell in col) for col in cols]
def fmt(row: list[str]) -> str:
return " | ".join(cell.ljust(w) for cell, w in zip(row, widths))
print(fmt(headers))
print("-+-".join("-" * w for w in widths))
for row in data:
print(fmt(row))
def log_versions() -> None:
"""Log torch/helion/triton versions at the head of the output."""
from importlib.metadata import PackageNotFoundError, version
def pkg_version(name: str) -> str:
try:
return version(name)
except PackageNotFoundError:
return "not installed"
logger.info("torch: %s", torch.__version__)
logger.info("helion: %s", pkg_version("helion"))
logger.info("triton: %s", getattr(triton, "__version__", pkg_version("triton")))
def list_kernels() -> None:
kernels = get_registered_kernels()
if not kernels:
print("No Helion kernels found in registry.")
return
print("Available Helion kernels:")
print("=" * 50)
for name in sorted(kernels.keys()):
cuda = CUDA_BASELINE_OPS.get(name)
suffix = "" if cuda else " (no CUDA baseline mapping)"
print(f" {name}{suffix}")
print(f"\nTotal: {len(kernels)} kernels")
def check_requirements() -> bool:
if not torch.accelerator.is_available():
logger.error("CUDA is not available. Helion benchmarking requires GPU.")
return False
if not has_helion():
logger.error("Helion is not installed. Please install Helion package.")
return False
return True
def make_cuda_baseline(kernel_name: str) -> Callable:
"""Return a callable invoking the CUDA op mapped to ``kernel_name``.
The Helion kernel and its CUDA op share the same argument interface, so the
input tuple is forwarded verbatim.
"""
cuda_op_name = CUDA_BASELINE_OPS.get(kernel_name)
if cuda_op_name is None:
logger.error(
"No CUDA baseline mapping for kernel '%s'. Add an entry to "
"CUDA_BASELINE_OPS in %s (mapping the kernel name to its "
"torch.ops._C.<op> name), or benchmark with --baseline torch.",
kernel_name,
__file__,
)
sys.exit(1)
cuda_op = getattr(torch.ops._C, cuda_op_name, None)
if cuda_op is None:
logger.error(
"torch.ops._C.%s is not available. Ensure the vLLM C extension is "
"built and loaded.",
cuda_op_name,
)
sys.exit(1)
return cuda_op
def make_eager_baseline(kernel_name: str) -> Callable:
"""Return the kernel's registered native-torch reference."""
wrapper = get_kernel_by_name(kernel_name)
settings = wrapper.helion_settings
baseline_fn = getattr(settings, "autotune_baseline_fn", None)
if baseline_fn is None:
logger.error(
"Kernel '%s' has no autotune_baseline_fn in its helion_settings, so "
"the 'autotune' baseline is unavailable. Register one via "
"register_kernel(..., helion_settings=helion.Settings("
"autotune_baseline_fn=...)), or benchmark with --baseline cuda.",
kernel_name,
)
sys.exit(1)
return baseline_fn
def make_autotune_baseline(kernel_name: str) -> Callable:
"""Return the kernel's autotune baseline wrapped in ``torch.compile``."""
baseline_fn = make_eager_baseline(kernel_name)
return torch.compile(
baseline_fn,
fullgraph=True,
dynamic=False,
backend="inductor",
options=_TORCH_COMPILE_OPTIONS,
)
def make_correctness_baseline(
kernel_name: str,
timed_baseline_fn: Callable,
numerics_with_perf_baseline: bool,
) -> Callable:
"""Choose the numerical reference independently from the timed baseline."""
if not numerics_with_perf_baseline:
logger.info(
"Using the eager reference for '%s' correctness",
kernel_name,
)
return make_eager_baseline(kernel_name)
logger.info(
"Using the selected performance baseline for '%s' correctness",
kernel_name,
)
return timed_baseline_fn
def cleanup_gpu_resources() -> None:
try:
torch.accelerator.empty_cache()
gc.collect()
if hasattr(torch, "_dynamo"):
torch._dynamo.reset()
torch.accelerator.synchronize()
except Exception as e:
logger.warning("Failed to cleanup GPU resources: %s", e)
_REDUCERS: dict[str, Callable[[list[float]], float]] = {
"min": min,
"max": max,
"mean": statistics.fmean,
"median": statistics.median,
}
def _reduce(times: list[float], return_mode: str) -> float:
return _REDUCERS[return_mode](times)
def _assert_close(actual: object, expected: object, atol: float, rtol: float) -> None:
"""Compare pytrees, allowing the one-ULP FP8 variance used by kernel tests."""
actual_flat, actual_spec = tree_flatten(actual)
expected_flat, expected_spec = tree_flatten(expected)
if actual_spec != expected_spec:
raise AssertionError(
f"Output structure mismatch: {actual_spec} != {expected_spec}"
)
for actual_leaf, expected_leaf in zip(actual_flat, expected_flat, strict=True):
is_fp8 = isinstance(actual_leaf, torch.Tensor) and is_fp8_dtype(
actual_leaf.dtype
)
helion_assert_close(
actual_leaf,
expected_leaf,
atol=1 if is_fp8 else atol,
rtol=0 if is_fp8 else rtol,
)
def check_correctness(
kernel: Any,
baseline_fn: Callable,
inputs: tuple[Any, ...],
case: str,
) -> None:
"""Run one numerical comparison on copies separate from benchmark inputs."""
kernel_inputs = copy.deepcopy(inputs)
baseline_inputs = copy.deepcopy(inputs)
kernel_output = kernel(*kernel_inputs)
baseline_output = baseline_fn(*baseline_inputs)
settings = kernel.helion_settings
try:
custom_check = getattr(settings, "autotune_baseline_accuracy_check_fn", None)
if custom_check is not None:
custom_check(kernel_output, baseline_output)
custom_check(kernel_inputs, baseline_inputs)
return
configured_atol = getattr(settings, "autotune_baseline_atol", None)
configured_rtol = getattr(settings, "autotune_baseline_rtol", None)
atol = 1e-2 if configured_atol is None else configured_atol
rtol = 1e-2 if configured_rtol is None else configured_rtol
_assert_close(
kernel_output,
baseline_output,
atol=atol,
rtol=rtol,
)
_assert_close(
kernel_inputs,
baseline_inputs,
atol=atol,
rtol=rtol,
)
except AssertionError as e:
raise AssertionError(f"Numerics check failed for case {case}:\n{e}") from e
@dataclass
class CorrectnessResult:
"""Outcome of the numerics check for a single shape case."""
case: str
passed: bool
error: str | None = None
def check_kernel_correctness(
kernel: Any,
baseline_fn: Callable,
inputs_dict: dict[Any, tuple[Any, ...]] | None = None,
) -> list[CorrectnessResult]:
"""Run the per-shape numerics check for a kernel, continuing past failures.
Runs the same comparison as ``check_correctness`` for every shape case
produced by the kernel's input generator, but records the outcome per case
instead of raising on the first mismatch. This lets callers (e.g. a CI gate)
report every failing shape in one pass rather than aborting early.
Args:
kernel: The Helion kernel wrapper to check.
baseline_fn: Reference callable sharing the kernel's argument interface.
inputs_dict: Optional mapping of case key to input tuple. Defaults to
``kernel.get_inputs()``.
Returns:
One ``CorrectnessResult`` per shape case, in iteration order. A case that
raises (compile/run error or numerics mismatch) is marked
``passed=False`` with the exception text in ``error``; iteration
continues regardless. An empty input mapping returns an empty list, which
the CLI reports as a skipped check.
"""
if inputs_dict is None:
inputs_dict = kernel.get_inputs()
results: list[CorrectnessResult] = []
for key, inputs in inputs_dict.items():
case = str(key)
try:
check_correctness(kernel, baseline_fn, inputs, case)
except Exception as e: # noqa: BLE001 - any failure is recorded, not fatal
results.append(CorrectnessResult(case=case, passed=False, error=str(e)))
else:
results.append(CorrectnessResult(case=case, passed=True))
cleanup_gpu_resources()
return results
def do_bench_cudagraph_l2_clear(
fn: Callable, rep: int = 100, return_mode: str = "mean"
) -> float:
"""CUDA-graph benchmark that flushes the L2 cache before every call.
``triton.testing.do_bench_cudagraph`` captures back-to-back kernel launches
with a warm L2 cache, which over-estimates performance for memory-bound
kernels. This clears L2 (via triton's benchmark cache buffer) before each
call and subtracts the isolated cache-clear cost from the measured time.
Adapted from tritonbench's ``_do_bench_cudagraph_with_cache_clear`` using
only triton/torch primitives so no extra dependency is introduced.
"""
cache = triton.runtime.driver.active.get_empty_cache_for_benchmark()
clear_cache = cache.zero_
s = torch.Stream()
with s:
clear_cache()
fn()
start_event = torch.Event(enable_timing=True)
end_event = torch.Event(enable_timing=True)
start_event.record()
for _ in range(5):
clear_cache()
fn()
end_event.record()
torch.accelerator.synchronize()
estimate_ms = start_event.elapsed_time(end_event) / 5
n_repeat = 1000 if estimate_ms == 0 else max(1, int(rep / estimate_ms))
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for _ in range(n_repeat):
clear_cache()
fn()
clear_graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(clear_graph):
for _ in range(n_repeat):
clear_cache()
torch.accelerator.synchronize()
kernel_times = []
for _ in range(10):
start_event = torch.Event(enable_timing=True)
end_event = torch.Event(enable_timing=True)
start_event.record()
clear_graph.replay()
end_event.record()
torch.accelerator.synchronize()
clear_ms = start_event.elapsed_time(end_event) / n_repeat
start_event = torch.Event(enable_timing=True)
end_event = torch.Event(enable_timing=True)
start_event.record()
graph.replay()
end_event.record()
torch.accelerator.synchronize()
total_ms = start_event.elapsed_time(end_event) / n_repeat
kernel_times.append(total_ms - clear_ms)
return _reduce(kernel_times, return_mode)
@torch.inference_mode()
def benchmark(
kernel_name: str,
baseline_fn: Callable,
correctness_fn: Callable,
repeat: int,
cudagraph: bool,
return_mode: str,
) -> list[Row]:
kernel = get_kernel_by_name(kernel_name)
# do_bench already flushes L2 per call; do_bench_cudagraph does not, so use
# the cache-clearing variant to avoid warm-L2 over-estimates.
benchmark_fn = do_bench_cudagraph_l2_clear if cudagraph else triton.testing.do_bench
inputs_dict = kernel.get_inputs()
rows: list[Row] = []
for key, inputs in inputs_dict.items():
logger.info("Benchmarking case %s", key)
check_correctness(kernel, correctness_fn, inputs, str(key))
logger.info("Numerics check passed for case %s", key)
# Kernels may mutate their inputs in place; give each side its own copy.
kernel_inputs = copy.deepcopy(inputs)
baseline_inputs = copy.deepcopy(inputs)
kernel_latency = benchmark_fn(
lambda kernel_inputs=kernel_inputs: kernel(*kernel_inputs),
rep=repeat,
return_mode=return_mode,
)
baseline_latency = benchmark_fn(
lambda baseline_inputs=baseline_inputs: baseline_fn(*baseline_inputs),
rep=repeat,
return_mode=return_mode,
)
rows.append(
Row(
case=str(key),
baseline_ms=baseline_latency,
kernel_ms=kernel_latency,
speedup_x=baseline_latency / kernel_latency,
)
)
cleanup_gpu_resources()
return rows
def main() -> None:
parser = argparse.ArgumentParser(
description="Benchmark a Helion kernel against a baseline",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__.split("Usage:")[1] if "Usage:" in __doc__ else "",
)
parser.add_argument(
"--list",
action="store_true",
help="List available Helion kernels and exit",
)
parser.add_argument(
"--kernel",
type=str,
help="Name of the single Helion kernel to benchmark",
)
parser.add_argument(
"--repeat",
type=int,
default=100,
help="Number of benchmark repetitions (default: 100)",
)
parser.add_argument(
"--no-cudagraph",
dest="cudagraph",
action="store_false",
help="Disable CUDA graph mode (enabled by default)",
)
parser.add_argument(
"--baseline",
choices=["cuda", "autotune"],
default="autotune",
help=(
"Performance baseline: 'autotune' uses the kernel's "
"autotune_baseline_fn under torch.compile; 'cuda' uses the mapped "
"torch.ops._C op (default: autotune)"
),
)
parser.add_argument(
"--return-mode",
choices=["min", "max", "mean", "median"],
default="mean",
help="Statistic to report from the benchmark samples (default: mean)",
)
parser.add_argument(
"--output",
type=str,
help="Path to save benchmark results as JSON (default: log only)",
)
parser.add_argument(
"--numerics-only",
action="store_true",
help="Only run the per-case numerics check; skip timing and reporting",
)
parser.add_argument(
"--numerics-with-perf-baseline",
action="store_true",
help=(
"Compare numerics against the selected performance baseline instead "
"of the eager reference"
),
)
args = parser.parse_args()
log_versions()
import_all_kernels()
if args.list:
list_kernels()
return
if not args.kernel:
parser.error("--kernel is required (or use --list to see available kernels)")
kernels = get_registered_kernels()
if args.kernel not in kernels:
logger.error("Kernel '%s' not found in registry.", args.kernel)
logger.error("Available kernels: %s", sorted(kernels.keys()))
sys.exit(1)
wrapper = kernels[args.kernel]
if wrapper._disabled:
logger.error(
"Kernel '%s' is disabled: %s",
args.kernel,
wrapper._disabled_reason,
)
sys.exit(1)
if not check_requirements():
sys.exit(1)
with default_vllm_config():
if args.baseline == "cuda":
baseline_fn = make_cuda_baseline(args.kernel)
else:
baseline_fn = make_autotune_baseline(args.kernel)
correctness_fn = make_correctness_baseline(
args.kernel,
baseline_fn,
args.numerics_with_perf_baseline,
)
if args.numerics_only:
results = check_kernel_correctness(wrapper, correctness_fn)
if not results:
logger.warning(
"No input cases generated for '%s'; skipping numerics check",
args.kernel,
)
return
for r in results:
if r.passed:
logger.info("Numerics check passed for case %s", r.case)
else:
logger.error(
"Numerics check FAILED for case %s: %s", r.case, r.error
)
failed = [r for r in results if not r.passed]
if failed:
logger.error(
"%d/%d case(s) failed numerics for '%s'",
len(failed),
len(results),
args.kernel,
)
sys.exit(1)
logger.info("Numerics check passed for all cases of '%s'", args.kernel)
return
rows = benchmark(
args.kernel,
baseline_fn,
correctness_fn,
args.repeat,
args.cudagraph,
args.return_mode,
)
print_table(rows)
if args.output:
with open(args.output, "w") as f:
json.dump(
{
"kernel": args.kernel,
"baseline": args.baseline,
"numerics_baseline": (
args.baseline if args.numerics_with_perf_baseline else "eager"
),
"cudagraph": args.cudagraph,
"repeat": args.repeat,
"return_mode": args.return_mode,
"results": [asdict(r) for r in rows],
},
f,
indent=2,
)
logger.info("Saved results to %s", args.output)
if __name__ == "__main__":
main()
+33
View File
@@ -0,0 +1,33 @@
#!/bin/bash
# Build vLLM Rust artifacts and install them into the vllm package.
# Usage: ./tools/build_rust.sh [--debug]
#
# By default builds in release mode. Pass --debug for faster compile times
# during development.
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
# Read the required toolchain from rust-toolchain.toml.
TOOLCHAIN=$(grep '^channel' "$REPO_ROOT/rust-toolchain.toml" | sed 's/.*= *"\(.*\)"/\1/')
# Ensure rustup and the required toolchain are available.
if ! command -v rustup &>/dev/null; then
echo "rustup not found, installing..."
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none
source "$HOME/.cargo/env"
fi
if ! rustup run "$TOOLCHAIN" rustc --version &>/dev/null; then
echo "Installing Rust toolchain: $TOOLCHAIN"
rustup toolchain install "$TOOLCHAIN"
fi
if [[ "${1:-}" == "--debug" ]]; then
PROFILE_ARG="--debug"
else
PROFILE_ARG="--release"
fi
python3 "$REPO_ROOT/tools/build_rust.py" "$PROFILE_ARG"
+14
View File
@@ -0,0 +1,14 @@
MD007:
indent: 4
MD013: false
MD024:
siblings_only: true
MD025:
# Allow front matter title to be different from the first heading in the document.
front_matter_title: ""
MD031:
list_items: false
MD033: false
MD046: false
MD052: false
MD059: false
+9
View File
@@ -0,0 +1,9 @@
# rules currently disabled:
#
# SC1091 (info): Not following: <sourced file> was not specified as input (see shellcheck -x)
# SC2004 (style): $/${} is unnecessary on arithmetic variables.
# SC2129 (style): Consider using { cmd1; cmd2; } >> file instead of individual redirects.
# SC2155 (warning): Declare and assign separately to avoid masking return values.
# SC2164 (warning): Use 'cd ... || exit' or 'cd ... || return' in case cd fails.
#
disable=SC1091,SC2004,SC2129,SC2155,SC2164
+56
View File
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import glob
import sys
# Only strip targeted libraries when checking prefix
TORCH_LIB_PREFIXES = (
# requirements/*.txt/in
"torch=",
"torchvision=",
"torchaudio=",
"torchcodec=",
# pyproject.toml
'"torch =',
'"torchvision =',
'"torchaudio =',
'"torchcodec =',
)
def main(argv):
parser = argparse.ArgumentParser(
description="Strip torch lib requirements to use installed version."
)
parser.add_argument(
"--prefix",
action="store_true",
help="Strip prefix matches only (default: False)",
)
args = parser.parse_args(argv)
for file in (
*glob.glob("requirements/**/*.txt", recursive=True),
*glob.glob("requirements/**/*.in", recursive=True),
*glob.glob("pyproject.toml"),
):
with open(file) as f:
lines = f.readlines()
if "torch" in "".join(lines).lower():
with open(file, "w") as f:
for line in lines:
if (
args.prefix
and not line.lower().strip().startswith(TORCH_LIB_PREFIXES)
or not args.prefix
and "torch" not in line.lower()
):
f.write(line)
else:
print(f">>> removed from {file}:", line.strip())
if __name__ == "__main__":
main(sys.argv[1:])