[Refactor] Move auxiliary files out of the repository root (#58572)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -0,0 +1,438 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Autotune registered Helion kernels for optimal configurations.
|
||||
|
||||
Usage:
|
||||
# Autotune all registered kernels
|
||||
python tools/autotune_helion_kernels.py
|
||||
|
||||
# Autotune specific kernel
|
||||
python tools/autotune_helion_kernels.py --kernels silu_mul_fp8
|
||||
|
||||
# Autotune multiple kernels
|
||||
python tools/autotune_helion_kernels.py --kernels silu_mul_fp8 rms_norm_fp8
|
||||
|
||||
# Force re-autotuning
|
||||
python tools/autotune_helion_kernels.py --force
|
||||
|
||||
# List available kernels
|
||||
python tools/autotune_helion_kernels.py --list
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
try:
|
||||
import helion
|
||||
|
||||
from vllm.kernels.helion import (
|
||||
ConfigManager,
|
||||
get_kernel_by_name,
|
||||
get_registered_kernels,
|
||||
)
|
||||
from vllm.kernels.helion.ops import import_all_kernels
|
||||
from vllm.kernels.helion.utils import get_canonical_gpu_name
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import has_helion
|
||||
except ImportError as e:
|
||||
print(f"Error importing vLLM: {e}")
|
||||
print("Please ensure vLLM is installed and in your Python path")
|
||||
sys.exit(1)
|
||||
|
||||
logger = init_logger("vllm.scripts.autotune_helion_kernels")
|
||||
|
||||
|
||||
@dataclass
|
||||
class AutotuneResult:
|
||||
status: str # "success" | "partial" | "error" | "skipped"
|
||||
successful: int
|
||||
failed: int
|
||||
configs: dict[str, "helion.Config"]
|
||||
message: str = ""
|
||||
|
||||
|
||||
def list_kernels() -> None:
|
||||
kernels = get_registered_kernels()
|
||||
|
||||
if not kernels:
|
||||
print("No Helion kernels found in registry.")
|
||||
return
|
||||
|
||||
print("Available Helion kernels:")
|
||||
print("=" * 50)
|
||||
|
||||
for name in sorted(kernels.keys()):
|
||||
print(f" {name}")
|
||||
|
||||
print(f"\nTotal: {len(kernels)} kernels")
|
||||
|
||||
|
||||
def check_requirements() -> bool:
|
||||
if not torch.cuda.is_available():
|
||||
logger.error("CUDA is not available. Helion autotuning requires GPU.")
|
||||
return False
|
||||
|
||||
if not has_helion():
|
||||
logger.error("Helion is not installed. Please install Helion package.")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def autotune_kernel(
|
||||
kernel_name: str,
|
||||
platform: str,
|
||||
config_manager: ConfigManager,
|
||||
force: bool = False,
|
||||
autotune_effort: str = "quick",
|
||||
) -> AutotuneResult:
|
||||
logger.debug(
|
||||
"Starting autotune for kernel '%s' with effort='%s'",
|
||||
kernel_name,
|
||||
autotune_effort,
|
||||
)
|
||||
kernel_wrapper = get_kernel_by_name(kernel_name)
|
||||
if kernel_wrapper is None:
|
||||
error_msg = f"Kernel '{kernel_name}' not found in registry"
|
||||
logger.error(error_msg)
|
||||
return AutotuneResult(
|
||||
status="error",
|
||||
message=error_msg,
|
||||
successful=0,
|
||||
failed=0,
|
||||
configs={},
|
||||
)
|
||||
|
||||
try:
|
||||
with FakeTensorMode():
|
||||
all_config_keys = list(kernel_wrapper.get_inputs().keys())
|
||||
except NotImplementedError:
|
||||
error_msg = f"Kernel '{kernel_name}' has no input generator registered"
|
||||
logger.error(error_msg)
|
||||
return AutotuneResult(
|
||||
status="error",
|
||||
message=error_msg,
|
||||
successful=0,
|
||||
failed=0,
|
||||
configs={},
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
"Autotuning kernel '%s' for platform '%s' with %d configs",
|
||||
kernel_name,
|
||||
platform,
|
||||
len(all_config_keys),
|
||||
)
|
||||
|
||||
if not force:
|
||||
existing_configs = config_manager.get_platform_configs(
|
||||
kernel_name, platform
|
||||
)
|
||||
keys_to_autotune = []
|
||||
for config_key in all_config_keys:
|
||||
if config_key in existing_configs:
|
||||
logger.debug(
|
||||
"Config '%s' already exists for platform '%s', skipping",
|
||||
config_key,
|
||||
platform,
|
||||
)
|
||||
else:
|
||||
keys_to_autotune.append(config_key)
|
||||
else:
|
||||
logger.debug("Force mode enabled, will re-autotune all configs")
|
||||
keys_to_autotune = all_config_keys
|
||||
|
||||
if not keys_to_autotune:
|
||||
logger.info(
|
||||
"All configs already exist for kernel '%s' on platform '%s'. "
|
||||
"Use --force to re-autotune.",
|
||||
kernel_name,
|
||||
platform,
|
||||
)
|
||||
return AutotuneResult(
|
||||
status="skipped",
|
||||
message="All configs already exist",
|
||||
successful=0,
|
||||
failed=0,
|
||||
configs={},
|
||||
)
|
||||
|
||||
inputs_dict = kernel_wrapper.get_inputs()
|
||||
configs_to_autotune = {k: inputs_dict[k] for k in keys_to_autotune}
|
||||
|
||||
total_start_time = time.time()
|
||||
autotuned_configs = {}
|
||||
failed_configs = []
|
||||
|
||||
for config_key, inputs in configs_to_autotune.items():
|
||||
logger.info("Autotuning config: %s", config_key)
|
||||
logger.debug(
|
||||
"Input shapes: %s",
|
||||
[getattr(inp, "shape", type(inp).__name__) for inp in inputs],
|
||||
)
|
||||
|
||||
try:
|
||||
config_start_time = time.time()
|
||||
config = kernel_wrapper.run_autotune(inputs, autotune_effort)
|
||||
config_duration = time.time() - config_start_time
|
||||
|
||||
# Save immediately for checkpointing
|
||||
config_manager.save_configs(kernel_name, platform, {config_key: config})
|
||||
|
||||
autotuned_configs[config_key] = config
|
||||
logger.debug("Config details: %s", config)
|
||||
|
||||
logger.info(
|
||||
"✓ Autotuned and saved config '%s' (%.2fs)",
|
||||
config_key,
|
||||
config_duration,
|
||||
)
|
||||
|
||||
except (RuntimeError, ValueError, OSError) as e:
|
||||
logger.exception(
|
||||
"Failed to autotune config '%s': %s",
|
||||
config_key,
|
||||
e,
|
||||
)
|
||||
failed_configs.append(config_key)
|
||||
|
||||
total_duration = time.time() - total_start_time
|
||||
successful = len(autotuned_configs)
|
||||
failed = len(failed_configs)
|
||||
|
||||
logger.info(
|
||||
"Completed autotuning for kernel '%s': %d successful, %d failed (%.2fs)",
|
||||
kernel_name,
|
||||
successful,
|
||||
failed,
|
||||
total_duration,
|
||||
)
|
||||
|
||||
status = "success" if failed == 0 else "partial"
|
||||
return AutotuneResult(
|
||||
status=status,
|
||||
successful=successful,
|
||||
failed=failed,
|
||||
configs=autotuned_configs,
|
||||
)
|
||||
|
||||
except (KeyError, RuntimeError, ValueError, OSError) as e:
|
||||
error_msg = f"Unexpected error: {e}"
|
||||
logger.exception("Failed to autotune kernel '%s': %s", kernel_name, e)
|
||||
return AutotuneResult(
|
||||
status="error",
|
||||
message=error_msg,
|
||||
successful=0,
|
||||
failed=0,
|
||||
configs={},
|
||||
)
|
||||
|
||||
|
||||
def summarize_results(results: dict[str, AutotuneResult]) -> bool:
|
||||
logger.info("=" * 50)
|
||||
logger.info("Autotuning Results Summary")
|
||||
logger.info("=" * 50)
|
||||
|
||||
total_successful = 0
|
||||
total_failed = 0
|
||||
success_kernels = []
|
||||
partial_kernels = []
|
||||
error_kernels = []
|
||||
skipped_kernels = []
|
||||
|
||||
for kernel_name, result in results.items():
|
||||
total_successful += result.successful
|
||||
total_failed += result.failed
|
||||
|
||||
if result.status == "success":
|
||||
success_kernels.append(f"{kernel_name} ({result.successful} configs)")
|
||||
logger.info("✓ %s: %d configs successful", kernel_name, result.successful)
|
||||
elif result.status == "partial":
|
||||
partial_kernels.append(
|
||||
f"{kernel_name} ({result.successful} ok, {result.failed} failed)"
|
||||
)
|
||||
logger.warning(
|
||||
"⚠ %s: %d successful, %d failed",
|
||||
kernel_name,
|
||||
result.successful,
|
||||
result.failed,
|
||||
)
|
||||
elif result.status == "error":
|
||||
error_kernels.append(f"{kernel_name}: {result.message or 'Unknown error'}")
|
||||
logger.error("✗ %s: %s", kernel_name, result.message or "Unknown error")
|
||||
elif result.status == "skipped":
|
||||
skipped_kernels.append(f"{kernel_name}: {result.message or 'Skipped'}")
|
||||
logger.info("- %s: %s", kernel_name, result.message or "Skipped")
|
||||
|
||||
logger.info("=" * 50)
|
||||
logger.info(
|
||||
"Summary: %d total configs (%d successful, %d failed)",
|
||||
total_successful + total_failed,
|
||||
total_successful,
|
||||
total_failed,
|
||||
)
|
||||
logger.info(
|
||||
"Kernels: %d success, %d partial, %d error, %d skipped",
|
||||
len(success_kernels),
|
||||
len(partial_kernels),
|
||||
len(error_kernels),
|
||||
len(skipped_kernels),
|
||||
)
|
||||
|
||||
has_failures = bool(error_kernels or partial_kernels)
|
||||
|
||||
if not has_failures:
|
||||
if total_successful > 0:
|
||||
logger.info("All configs autotuned successfully!")
|
||||
else:
|
||||
logger.info("No new configs were generated (all may already exist)")
|
||||
|
||||
return not has_failures
|
||||
|
||||
|
||||
def get_kernels_to_autotune(requested_kernels: list[str] | None) -> list[str]:
|
||||
all_kernels = get_registered_kernels()
|
||||
if not all_kernels:
|
||||
logger.error("No Helion kernels found in registry")
|
||||
sys.exit(1)
|
||||
|
||||
if not requested_kernels:
|
||||
return list(all_kernels.keys())
|
||||
|
||||
if len(requested_kernels) != len(set(requested_kernels)):
|
||||
duplicates = [
|
||||
k for k in set(requested_kernels) if requested_kernels.count(k) > 1
|
||||
]
|
||||
logger.error("Duplicate kernel names in --kernels flag: %s", duplicates)
|
||||
sys.exit(1)
|
||||
|
||||
kernels_to_autotune = []
|
||||
missing_kernels = []
|
||||
|
||||
for kernel_name in requested_kernels:
|
||||
if kernel_name in all_kernels:
|
||||
kernels_to_autotune.append(kernel_name)
|
||||
else:
|
||||
missing_kernels.append(kernel_name)
|
||||
|
||||
if missing_kernels:
|
||||
logger.error("Kernel(s) not found: %s", missing_kernels)
|
||||
logger.error("Available kernels: %s", list(all_kernels.keys()))
|
||||
sys.exit(1)
|
||||
|
||||
return kernels_to_autotune
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Autotune Helion kernels",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__.split("Usage:")[1] if "Usage:" in __doc__ else "",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--kernels",
|
||||
nargs="+",
|
||||
help="Kernel(s) to autotune (default: all kernels)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--config-dir",
|
||||
type=str,
|
||||
help="Config directory for config files (default: vLLM helion configs dir)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--list",
|
||||
action="store_true",
|
||||
help="List available Helion kernels and exit",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Force re-autotuning even if configs already exist for the "
|
||||
"platform and config keys"
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--autotune-effort",
|
||||
type=str,
|
||||
default="quick",
|
||||
help=(
|
||||
"Helion autotune effort level: 'quick' (smaller search) or "
|
||||
"'full' (full search budget) (default: quick)"
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
help="Enable verbose logging",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# import all helion kernels
|
||||
import_all_kernels()
|
||||
|
||||
import logging
|
||||
|
||||
if args.verbose:
|
||||
logging.getLogger("vllm").setLevel(logging.DEBUG)
|
||||
logger.debug("Verbose mode enabled")
|
||||
logger.debug("Arguments: %s", vars(args))
|
||||
else:
|
||||
logging.getLogger("vllm").setLevel(logging.INFO)
|
||||
|
||||
if args.list:
|
||||
list_kernels()
|
||||
return
|
||||
|
||||
if not check_requirements():
|
||||
sys.exit(1)
|
||||
|
||||
platform = get_canonical_gpu_name()
|
||||
logger.info("Detected GPU platform: %s", platform)
|
||||
|
||||
config_manager = (
|
||||
ConfigManager(args.config_dir) if args.config_dir else ConfigManager()
|
||||
)
|
||||
|
||||
try:
|
||||
config_manager.ensure_base_dir_writable()
|
||||
except OSError as e:
|
||||
logger.error("Failed to access config directory: %s", e)
|
||||
sys.exit(1)
|
||||
|
||||
kernels_to_autotune = get_kernels_to_autotune(args.kernels)
|
||||
|
||||
logger.info(
|
||||
"Will autotune %d kernel(s) for platform '%s': %s",
|
||||
len(kernels_to_autotune),
|
||||
platform,
|
||||
kernels_to_autotune,
|
||||
)
|
||||
|
||||
results = {}
|
||||
for kernel_name in kernels_to_autotune:
|
||||
result = autotune_kernel(
|
||||
kernel_name, platform, config_manager, args.force, args.autotune_effort
|
||||
)
|
||||
results[kernel_name] = result
|
||||
|
||||
success = summarize_results(results)
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,696 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Benchmark a registered Helion kernel against a baseline.
|
||||
|
||||
For each input case produced by the kernel's registered input generator, this
|
||||
checks the Helion kernel's numerics once against its eager reference, measures
|
||||
its latency against a chosen performance baseline, then reports the speedup.
|
||||
Use ``--numerics-with-perf-baseline`` to check numerics against the performance
|
||||
baseline instead.
|
||||
|
||||
Two baselines are supported (``--baseline``):
|
||||
|
||||
- ``autotune`` (default): the kernel's autotuning baseline
|
||||
(``helion_settings.autotune_baseline_fn``), wrapped in ``torch.compile``
|
||||
(inductor). This is the native-torch reference used by kernel autotuning
|
||||
and correctness unit tests.
|
||||
- ``cuda``: the corresponding hand-written CUDA op (``torch.ops._C.*``). The
|
||||
mapping from Helion kernel name to CUDA op lives in ``CUDA_BASELINE_OPS``
|
||||
below. Every Helion kernel shares the same argument interface as its CUDA
|
||||
counterpart, so inputs are forwarded verbatim.
|
||||
|
||||
Usage:
|
||||
# List available kernels
|
||||
python tools/benchmark_helion_kernels.py --list
|
||||
|
||||
# Benchmark a kernel against the autotune baseline (default)
|
||||
python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant
|
||||
|
||||
# Benchmark against the CUDA baseline
|
||||
python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
|
||||
--baseline cuda
|
||||
|
||||
# Check numerics against the performance baseline instead of eager
|
||||
python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
|
||||
--baseline cuda --numerics-with-perf-baseline
|
||||
|
||||
# Disable CUDA graph capture and save results
|
||||
python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
|
||||
--no-cudagraph --output results.json
|
||||
|
||||
# Only verify numerics, skipping the timing runs
|
||||
python tools/benchmark_helion_kernels.py --kernel per_token_group_fp8_quant \\
|
||||
--numerics-only
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import gc
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch.utils._pytree import tree_flatten
|
||||
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
try:
|
||||
from helion.autotuner.accuracy import assert_close as helion_assert_close
|
||||
from helion.autotuner.accuracy import is_fp8_dtype
|
||||
|
||||
from vllm.benchmarks.lib.utils import default_vllm_config
|
||||
from vllm.kernels.helion import get_kernel_by_name, get_registered_kernels
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import has_helion
|
||||
except ImportError as e:
|
||||
print(f"Error importing vLLM: {e}")
|
||||
print("Please ensure vLLM is installed and in your Python path")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def import_all_kernels() -> None:
|
||||
"""Trigger Helion op registration, tolerating cross-version name drift.
|
||||
|
||||
Current vLLM registers every Helion kernel as a side effect of importing
|
||||
``vllm.kernels.helion.ops``; some builds instead expose an explicit importer
|
||||
whose name has drifted (``import_all_kernels`` / ``import_all_ops``). Call
|
||||
whichever exists; if none does, importing the module already registered
|
||||
them.
|
||||
"""
|
||||
try:
|
||||
import vllm.kernels.helion.ops as ops
|
||||
except ImportError:
|
||||
return
|
||||
for fn_name in ("import_all_kernels", "import_all_ops"):
|
||||
fn = getattr(ops, fn_name, None)
|
||||
if callable(fn):
|
||||
fn()
|
||||
return
|
||||
|
||||
|
||||
logger = init_logger("vllm.scripts.benchmark_helion_kernels")
|
||||
|
||||
|
||||
# Maps a Helion kernel name to the CUDA op (attribute on ``torch.ops._C``) that
|
||||
# implements the same operation. Helion kernels share the CUDA op's argument
|
||||
# interface, so the kernel's input tuple is forwarded verbatim. Add an entry
|
||||
# here when introducing a new kernel whose baseline should be the CUDA op.
|
||||
CUDA_BASELINE_OPS: dict[str, str] = {
|
||||
"dynamic_per_token_scaled_fp8_quant": "dynamic_per_token_scaled_fp8_quant",
|
||||
"fused_qk_norm_rope": "fused_qk_norm_rope",
|
||||
"per_token_group_fp8_quant": "per_token_group_fp8_quant",
|
||||
"rms_norm_dynamic_per_token_quant": "rms_norm_dynamic_per_token_quant",
|
||||
"rms_norm_per_block_quant": "rms_norm_per_block_quant",
|
||||
"silu_and_mul_per_block_quant": "silu_and_mul_per_block_quant",
|
||||
"scaled_mm": "cutlass_scaled_mm",
|
||||
}
|
||||
|
||||
# torch.compile options for the torch baseline, mirroring how these kernels are
|
||||
# compiled inside vLLM.
|
||||
_TORCH_COMPILE_OPTIONS: dict[str, bool] = {
|
||||
"enable_auto_functionalized_v2": False,
|
||||
"size_asserts": False,
|
||||
"alignment_asserts": False,
|
||||
"scalar_asserts": False,
|
||||
"combo_kernels": True,
|
||||
"benchmark_combo_kernel": True,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Row:
|
||||
case: str
|
||||
baseline_ms: float
|
||||
kernel_ms: float
|
||||
speedup_x: float
|
||||
|
||||
|
||||
def print_table(rows: list[Row]) -> None:
|
||||
headers = ["case", "baseline_ms", "kernel_ms", "speedup(x)"]
|
||||
|
||||
data = [
|
||||
[
|
||||
r.case,
|
||||
f"{r.baseline_ms:.3f}",
|
||||
f"{r.kernel_ms:.3f}",
|
||||
f"{r.speedup_x:.3f}",
|
||||
]
|
||||
for r in rows
|
||||
]
|
||||
|
||||
cols = list(zip(*([headers] + data)))
|
||||
widths = [max(len(cell) for cell in col) for col in cols]
|
||||
|
||||
def fmt(row: list[str]) -> str:
|
||||
return " | ".join(cell.ljust(w) for cell, w in zip(row, widths))
|
||||
|
||||
print(fmt(headers))
|
||||
print("-+-".join("-" * w for w in widths))
|
||||
for row in data:
|
||||
print(fmt(row))
|
||||
|
||||
|
||||
def log_versions() -> None:
|
||||
"""Log torch/helion/triton versions at the head of the output."""
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
|
||||
def pkg_version(name: str) -> str:
|
||||
try:
|
||||
return version(name)
|
||||
except PackageNotFoundError:
|
||||
return "not installed"
|
||||
|
||||
logger.info("torch: %s", torch.__version__)
|
||||
logger.info("helion: %s", pkg_version("helion"))
|
||||
logger.info("triton: %s", getattr(triton, "__version__", pkg_version("triton")))
|
||||
|
||||
|
||||
def list_kernels() -> None:
|
||||
kernels = get_registered_kernels()
|
||||
|
||||
if not kernels:
|
||||
print("No Helion kernels found in registry.")
|
||||
return
|
||||
|
||||
print("Available Helion kernels:")
|
||||
print("=" * 50)
|
||||
for name in sorted(kernels.keys()):
|
||||
cuda = CUDA_BASELINE_OPS.get(name)
|
||||
suffix = "" if cuda else " (no CUDA baseline mapping)"
|
||||
print(f" {name}{suffix}")
|
||||
print(f"\nTotal: {len(kernels)} kernels")
|
||||
|
||||
|
||||
def check_requirements() -> bool:
|
||||
if not torch.accelerator.is_available():
|
||||
logger.error("CUDA is not available. Helion benchmarking requires GPU.")
|
||||
return False
|
||||
if not has_helion():
|
||||
logger.error("Helion is not installed. Please install Helion package.")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def make_cuda_baseline(kernel_name: str) -> Callable:
|
||||
"""Return a callable invoking the CUDA op mapped to ``kernel_name``.
|
||||
|
||||
The Helion kernel and its CUDA op share the same argument interface, so the
|
||||
input tuple is forwarded verbatim.
|
||||
"""
|
||||
cuda_op_name = CUDA_BASELINE_OPS.get(kernel_name)
|
||||
if cuda_op_name is None:
|
||||
logger.error(
|
||||
"No CUDA baseline mapping for kernel '%s'. Add an entry to "
|
||||
"CUDA_BASELINE_OPS in %s (mapping the kernel name to its "
|
||||
"torch.ops._C.<op> name), or benchmark with --baseline torch.",
|
||||
kernel_name,
|
||||
__file__,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
cuda_op = getattr(torch.ops._C, cuda_op_name, None)
|
||||
if cuda_op is None:
|
||||
logger.error(
|
||||
"torch.ops._C.%s is not available. Ensure the vLLM C extension is "
|
||||
"built and loaded.",
|
||||
cuda_op_name,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
return cuda_op
|
||||
|
||||
|
||||
def make_eager_baseline(kernel_name: str) -> Callable:
|
||||
"""Return the kernel's registered native-torch reference."""
|
||||
wrapper = get_kernel_by_name(kernel_name)
|
||||
settings = wrapper.helion_settings
|
||||
baseline_fn = getattr(settings, "autotune_baseline_fn", None)
|
||||
if baseline_fn is None:
|
||||
logger.error(
|
||||
"Kernel '%s' has no autotune_baseline_fn in its helion_settings, so "
|
||||
"the 'autotune' baseline is unavailable. Register one via "
|
||||
"register_kernel(..., helion_settings=helion.Settings("
|
||||
"autotune_baseline_fn=...)), or benchmark with --baseline cuda.",
|
||||
kernel_name,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
return baseline_fn
|
||||
|
||||
|
||||
def make_autotune_baseline(kernel_name: str) -> Callable:
|
||||
"""Return the kernel's autotune baseline wrapped in ``torch.compile``."""
|
||||
baseline_fn = make_eager_baseline(kernel_name)
|
||||
|
||||
return torch.compile(
|
||||
baseline_fn,
|
||||
fullgraph=True,
|
||||
dynamic=False,
|
||||
backend="inductor",
|
||||
options=_TORCH_COMPILE_OPTIONS,
|
||||
)
|
||||
|
||||
|
||||
def make_correctness_baseline(
|
||||
kernel_name: str,
|
||||
timed_baseline_fn: Callable,
|
||||
numerics_with_perf_baseline: bool,
|
||||
) -> Callable:
|
||||
"""Choose the numerical reference independently from the timed baseline."""
|
||||
if not numerics_with_perf_baseline:
|
||||
logger.info(
|
||||
"Using the eager reference for '%s' correctness",
|
||||
kernel_name,
|
||||
)
|
||||
return make_eager_baseline(kernel_name)
|
||||
logger.info(
|
||||
"Using the selected performance baseline for '%s' correctness",
|
||||
kernel_name,
|
||||
)
|
||||
return timed_baseline_fn
|
||||
|
||||
|
||||
def cleanup_gpu_resources() -> None:
|
||||
try:
|
||||
torch.accelerator.empty_cache()
|
||||
gc.collect()
|
||||
if hasattr(torch, "_dynamo"):
|
||||
torch._dynamo.reset()
|
||||
torch.accelerator.synchronize()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to cleanup GPU resources: %s", e)
|
||||
|
||||
|
||||
_REDUCERS: dict[str, Callable[[list[float]], float]] = {
|
||||
"min": min,
|
||||
"max": max,
|
||||
"mean": statistics.fmean,
|
||||
"median": statistics.median,
|
||||
}
|
||||
|
||||
|
||||
def _reduce(times: list[float], return_mode: str) -> float:
|
||||
return _REDUCERS[return_mode](times)
|
||||
|
||||
|
||||
def _assert_close(actual: object, expected: object, atol: float, rtol: float) -> None:
|
||||
"""Compare pytrees, allowing the one-ULP FP8 variance used by kernel tests."""
|
||||
actual_flat, actual_spec = tree_flatten(actual)
|
||||
expected_flat, expected_spec = tree_flatten(expected)
|
||||
if actual_spec != expected_spec:
|
||||
raise AssertionError(
|
||||
f"Output structure mismatch: {actual_spec} != {expected_spec}"
|
||||
)
|
||||
|
||||
for actual_leaf, expected_leaf in zip(actual_flat, expected_flat, strict=True):
|
||||
is_fp8 = isinstance(actual_leaf, torch.Tensor) and is_fp8_dtype(
|
||||
actual_leaf.dtype
|
||||
)
|
||||
helion_assert_close(
|
||||
actual_leaf,
|
||||
expected_leaf,
|
||||
atol=1 if is_fp8 else atol,
|
||||
rtol=0 if is_fp8 else rtol,
|
||||
)
|
||||
|
||||
|
||||
def check_correctness(
|
||||
kernel: Any,
|
||||
baseline_fn: Callable,
|
||||
inputs: tuple[Any, ...],
|
||||
case: str,
|
||||
) -> None:
|
||||
"""Run one numerical comparison on copies separate from benchmark inputs."""
|
||||
kernel_inputs = copy.deepcopy(inputs)
|
||||
baseline_inputs = copy.deepcopy(inputs)
|
||||
|
||||
kernel_output = kernel(*kernel_inputs)
|
||||
baseline_output = baseline_fn(*baseline_inputs)
|
||||
|
||||
settings = kernel.helion_settings
|
||||
try:
|
||||
custom_check = getattr(settings, "autotune_baseline_accuracy_check_fn", None)
|
||||
if custom_check is not None:
|
||||
custom_check(kernel_output, baseline_output)
|
||||
custom_check(kernel_inputs, baseline_inputs)
|
||||
return
|
||||
|
||||
configured_atol = getattr(settings, "autotune_baseline_atol", None)
|
||||
configured_rtol = getattr(settings, "autotune_baseline_rtol", None)
|
||||
atol = 1e-2 if configured_atol is None else configured_atol
|
||||
rtol = 1e-2 if configured_rtol is None else configured_rtol
|
||||
_assert_close(
|
||||
kernel_output,
|
||||
baseline_output,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
)
|
||||
_assert_close(
|
||||
kernel_inputs,
|
||||
baseline_inputs,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
)
|
||||
except AssertionError as e:
|
||||
raise AssertionError(f"Numerics check failed for case {case}:\n{e}") from e
|
||||
|
||||
|
||||
@dataclass
|
||||
class CorrectnessResult:
|
||||
"""Outcome of the numerics check for a single shape case."""
|
||||
|
||||
case: str
|
||||
passed: bool
|
||||
error: str | None = None
|
||||
|
||||
|
||||
def check_kernel_correctness(
|
||||
kernel: Any,
|
||||
baseline_fn: Callable,
|
||||
inputs_dict: dict[Any, tuple[Any, ...]] | None = None,
|
||||
) -> list[CorrectnessResult]:
|
||||
"""Run the per-shape numerics check for a kernel, continuing past failures.
|
||||
|
||||
Runs the same comparison as ``check_correctness`` for every shape case
|
||||
produced by the kernel's input generator, but records the outcome per case
|
||||
instead of raising on the first mismatch. This lets callers (e.g. a CI gate)
|
||||
report every failing shape in one pass rather than aborting early.
|
||||
|
||||
Args:
|
||||
kernel: The Helion kernel wrapper to check.
|
||||
baseline_fn: Reference callable sharing the kernel's argument interface.
|
||||
inputs_dict: Optional mapping of case key to input tuple. Defaults to
|
||||
``kernel.get_inputs()``.
|
||||
|
||||
Returns:
|
||||
One ``CorrectnessResult`` per shape case, in iteration order. A case that
|
||||
raises (compile/run error or numerics mismatch) is marked
|
||||
``passed=False`` with the exception text in ``error``; iteration
|
||||
continues regardless. An empty input mapping returns an empty list, which
|
||||
the CLI reports as a skipped check.
|
||||
|
||||
"""
|
||||
if inputs_dict is None:
|
||||
inputs_dict = kernel.get_inputs()
|
||||
|
||||
results: list[CorrectnessResult] = []
|
||||
for key, inputs in inputs_dict.items():
|
||||
case = str(key)
|
||||
try:
|
||||
check_correctness(kernel, baseline_fn, inputs, case)
|
||||
except Exception as e: # noqa: BLE001 - any failure is recorded, not fatal
|
||||
results.append(CorrectnessResult(case=case, passed=False, error=str(e)))
|
||||
else:
|
||||
results.append(CorrectnessResult(case=case, passed=True))
|
||||
cleanup_gpu_resources()
|
||||
return results
|
||||
|
||||
|
||||
def do_bench_cudagraph_l2_clear(
|
||||
fn: Callable, rep: int = 100, return_mode: str = "mean"
|
||||
) -> float:
|
||||
"""CUDA-graph benchmark that flushes the L2 cache before every call.
|
||||
|
||||
``triton.testing.do_bench_cudagraph`` captures back-to-back kernel launches
|
||||
with a warm L2 cache, which over-estimates performance for memory-bound
|
||||
kernels. This clears L2 (via triton's benchmark cache buffer) before each
|
||||
call and subtracts the isolated cache-clear cost from the measured time.
|
||||
|
||||
Adapted from tritonbench's ``_do_bench_cudagraph_with_cache_clear`` using
|
||||
only triton/torch primitives so no extra dependency is introduced.
|
||||
"""
|
||||
cache = triton.runtime.driver.active.get_empty_cache_for_benchmark()
|
||||
clear_cache = cache.zero_
|
||||
|
||||
s = torch.Stream()
|
||||
with s:
|
||||
clear_cache()
|
||||
fn()
|
||||
|
||||
start_event = torch.Event(enable_timing=True)
|
||||
end_event = torch.Event(enable_timing=True)
|
||||
start_event.record()
|
||||
for _ in range(5):
|
||||
clear_cache()
|
||||
fn()
|
||||
end_event.record()
|
||||
torch.accelerator.synchronize()
|
||||
estimate_ms = start_event.elapsed_time(end_event) / 5
|
||||
n_repeat = 1000 if estimate_ms == 0 else max(1, int(rep / estimate_ms))
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
for _ in range(n_repeat):
|
||||
clear_cache()
|
||||
fn()
|
||||
|
||||
clear_graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(clear_graph):
|
||||
for _ in range(n_repeat):
|
||||
clear_cache()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
kernel_times = []
|
||||
for _ in range(10):
|
||||
start_event = torch.Event(enable_timing=True)
|
||||
end_event = torch.Event(enable_timing=True)
|
||||
start_event.record()
|
||||
clear_graph.replay()
|
||||
end_event.record()
|
||||
torch.accelerator.synchronize()
|
||||
clear_ms = start_event.elapsed_time(end_event) / n_repeat
|
||||
|
||||
start_event = torch.Event(enable_timing=True)
|
||||
end_event = torch.Event(enable_timing=True)
|
||||
start_event.record()
|
||||
graph.replay()
|
||||
end_event.record()
|
||||
torch.accelerator.synchronize()
|
||||
total_ms = start_event.elapsed_time(end_event) / n_repeat
|
||||
|
||||
kernel_times.append(total_ms - clear_ms)
|
||||
|
||||
return _reduce(kernel_times, return_mode)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def benchmark(
|
||||
kernel_name: str,
|
||||
baseline_fn: Callable,
|
||||
correctness_fn: Callable,
|
||||
repeat: int,
|
||||
cudagraph: bool,
|
||||
return_mode: str,
|
||||
) -> list[Row]:
|
||||
kernel = get_kernel_by_name(kernel_name)
|
||||
# do_bench already flushes L2 per call; do_bench_cudagraph does not, so use
|
||||
# the cache-clearing variant to avoid warm-L2 over-estimates.
|
||||
benchmark_fn = do_bench_cudagraph_l2_clear if cudagraph else triton.testing.do_bench
|
||||
|
||||
inputs_dict = kernel.get_inputs()
|
||||
rows: list[Row] = []
|
||||
|
||||
for key, inputs in inputs_dict.items():
|
||||
logger.info("Benchmarking case %s", key)
|
||||
|
||||
check_correctness(kernel, correctness_fn, inputs, str(key))
|
||||
logger.info("Numerics check passed for case %s", key)
|
||||
|
||||
# Kernels may mutate their inputs in place; give each side its own copy.
|
||||
kernel_inputs = copy.deepcopy(inputs)
|
||||
baseline_inputs = copy.deepcopy(inputs)
|
||||
|
||||
kernel_latency = benchmark_fn(
|
||||
lambda kernel_inputs=kernel_inputs: kernel(*kernel_inputs),
|
||||
rep=repeat,
|
||||
return_mode=return_mode,
|
||||
)
|
||||
baseline_latency = benchmark_fn(
|
||||
lambda baseline_inputs=baseline_inputs: baseline_fn(*baseline_inputs),
|
||||
rep=repeat,
|
||||
return_mode=return_mode,
|
||||
)
|
||||
|
||||
rows.append(
|
||||
Row(
|
||||
case=str(key),
|
||||
baseline_ms=baseline_latency,
|
||||
kernel_ms=kernel_latency,
|
||||
speedup_x=baseline_latency / kernel_latency,
|
||||
)
|
||||
)
|
||||
cleanup_gpu_resources()
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Benchmark a Helion kernel against a baseline",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__.split("Usage:")[1] if "Usage:" in __doc__ else "",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list",
|
||||
action="store_true",
|
||||
help="List available Helion kernels and exit",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--kernel",
|
||||
type=str,
|
||||
help="Name of the single Helion kernel to benchmark",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repeat",
|
||||
type=int,
|
||||
default=100,
|
||||
help="Number of benchmark repetitions (default: 100)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-cudagraph",
|
||||
dest="cudagraph",
|
||||
action="store_false",
|
||||
help="Disable CUDA graph mode (enabled by default)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline",
|
||||
choices=["cuda", "autotune"],
|
||||
default="autotune",
|
||||
help=(
|
||||
"Performance baseline: 'autotune' uses the kernel's "
|
||||
"autotune_baseline_fn under torch.compile; 'cuda' uses the mapped "
|
||||
"torch.ops._C op (default: autotune)"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--return-mode",
|
||||
choices=["min", "max", "mean", "median"],
|
||||
default="mean",
|
||||
help="Statistic to report from the benchmark samples (default: mean)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
help="Path to save benchmark results as JSON (default: log only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--numerics-only",
|
||||
action="store_true",
|
||||
help="Only run the per-case numerics check; skip timing and reporting",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--numerics-with-perf-baseline",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Compare numerics against the selected performance baseline instead "
|
||||
"of the eager reference"
|
||||
),
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
log_versions()
|
||||
|
||||
import_all_kernels()
|
||||
|
||||
if args.list:
|
||||
list_kernels()
|
||||
return
|
||||
|
||||
if not args.kernel:
|
||||
parser.error("--kernel is required (or use --list to see available kernels)")
|
||||
|
||||
kernels = get_registered_kernels()
|
||||
if args.kernel not in kernels:
|
||||
logger.error("Kernel '%s' not found in registry.", args.kernel)
|
||||
logger.error("Available kernels: %s", sorted(kernels.keys()))
|
||||
sys.exit(1)
|
||||
|
||||
wrapper = kernels[args.kernel]
|
||||
if wrapper._disabled:
|
||||
logger.error(
|
||||
"Kernel '%s' is disabled: %s",
|
||||
args.kernel,
|
||||
wrapper._disabled_reason,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
if not check_requirements():
|
||||
sys.exit(1)
|
||||
|
||||
with default_vllm_config():
|
||||
if args.baseline == "cuda":
|
||||
baseline_fn = make_cuda_baseline(args.kernel)
|
||||
else:
|
||||
baseline_fn = make_autotune_baseline(args.kernel)
|
||||
correctness_fn = make_correctness_baseline(
|
||||
args.kernel,
|
||||
baseline_fn,
|
||||
args.numerics_with_perf_baseline,
|
||||
)
|
||||
|
||||
if args.numerics_only:
|
||||
results = check_kernel_correctness(wrapper, correctness_fn)
|
||||
if not results:
|
||||
logger.warning(
|
||||
"No input cases generated for '%s'; skipping numerics check",
|
||||
args.kernel,
|
||||
)
|
||||
return
|
||||
for r in results:
|
||||
if r.passed:
|
||||
logger.info("Numerics check passed for case %s", r.case)
|
||||
else:
|
||||
logger.error(
|
||||
"Numerics check FAILED for case %s: %s", r.case, r.error
|
||||
)
|
||||
failed = [r for r in results if not r.passed]
|
||||
if failed:
|
||||
logger.error(
|
||||
"%d/%d case(s) failed numerics for '%s'",
|
||||
len(failed),
|
||||
len(results),
|
||||
args.kernel,
|
||||
)
|
||||
sys.exit(1)
|
||||
logger.info("Numerics check passed for all cases of '%s'", args.kernel)
|
||||
return
|
||||
|
||||
rows = benchmark(
|
||||
args.kernel,
|
||||
baseline_fn,
|
||||
correctness_fn,
|
||||
args.repeat,
|
||||
args.cudagraph,
|
||||
args.return_mode,
|
||||
)
|
||||
|
||||
print_table(rows)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "w") as f:
|
||||
json.dump(
|
||||
{
|
||||
"kernel": args.kernel,
|
||||
"baseline": args.baseline,
|
||||
"numerics_baseline": (
|
||||
args.baseline if args.numerics_with_perf_baseline else "eager"
|
||||
),
|
||||
"cudagraph": args.cudagraph,
|
||||
"repeat": args.repeat,
|
||||
"return_mode": args.return_mode,
|
||||
"results": [asdict(r) for r in rows],
|
||||
},
|
||||
f,
|
||||
indent=2,
|
||||
)
|
||||
logger.info("Saved results to %s", args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+33
@@ -0,0 +1,33 @@
|
||||
#!/bin/bash
|
||||
# Build vLLM Rust artifacts and install them into the vllm package.
|
||||
# Usage: ./tools/build_rust.sh [--debug]
|
||||
#
|
||||
# By default builds in release mode. Pass --debug for faster compile times
|
||||
# during development.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
# Read the required toolchain from rust-toolchain.toml.
|
||||
TOOLCHAIN=$(grep '^channel' "$REPO_ROOT/rust-toolchain.toml" | sed 's/.*= *"\(.*\)"/\1/')
|
||||
|
||||
# Ensure rustup and the required toolchain are available.
|
||||
if ! command -v rustup &>/dev/null; then
|
||||
echo "rustup not found, installing..."
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none
|
||||
source "$HOME/.cargo/env"
|
||||
fi
|
||||
|
||||
if ! rustup run "$TOOLCHAIN" rustc --version &>/dev/null; then
|
||||
echo "Installing Rust toolchain: $TOOLCHAIN"
|
||||
rustup toolchain install "$TOOLCHAIN"
|
||||
fi
|
||||
|
||||
if [[ "${1:-}" == "--debug" ]]; then
|
||||
PROFILE_ARG="--debug"
|
||||
else
|
||||
PROFILE_ARG="--release"
|
||||
fi
|
||||
|
||||
python3 "$REPO_ROOT/tools/build_rust.py" "$PROFILE_ARG"
|
||||
@@ -0,0 +1,14 @@
|
||||
MD007:
|
||||
indent: 4
|
||||
MD013: false
|
||||
MD024:
|
||||
siblings_only: true
|
||||
MD025:
|
||||
# Allow front matter title to be different from the first heading in the document.
|
||||
front_matter_title: ""
|
||||
MD031:
|
||||
list_items: false
|
||||
MD033: false
|
||||
MD046: false
|
||||
MD052: false
|
||||
MD059: false
|
||||
@@ -0,0 +1,9 @@
|
||||
# rules currently disabled:
|
||||
#
|
||||
# SC1091 (info): Not following: <sourced file> was not specified as input (see shellcheck -x)
|
||||
# SC2004 (style): $/${} is unnecessary on arithmetic variables.
|
||||
# SC2129 (style): Consider using { cmd1; cmd2; } >> file instead of individual redirects.
|
||||
# SC2155 (warning): Declare and assign separately to avoid masking return values.
|
||||
# SC2164 (warning): Use 'cd ... || exit' or 'cd ... || return' in case cd fails.
|
||||
#
|
||||
disable=SC1091,SC2004,SC2129,SC2155,SC2164
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import sys
|
||||
|
||||
# Only strip targeted libraries when checking prefix
|
||||
TORCH_LIB_PREFIXES = (
|
||||
# requirements/*.txt/in
|
||||
"torch=",
|
||||
"torchvision=",
|
||||
"torchaudio=",
|
||||
"torchcodec=",
|
||||
# pyproject.toml
|
||||
'"torch =',
|
||||
'"torchvision =',
|
||||
'"torchaudio =',
|
||||
'"torchcodec =',
|
||||
)
|
||||
|
||||
|
||||
def main(argv):
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Strip torch lib requirements to use installed version."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prefix",
|
||||
action="store_true",
|
||||
help="Strip prefix matches only (default: False)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
for file in (
|
||||
*glob.glob("requirements/**/*.txt", recursive=True),
|
||||
*glob.glob("requirements/**/*.in", recursive=True),
|
||||
*glob.glob("pyproject.toml"),
|
||||
):
|
||||
with open(file) as f:
|
||||
lines = f.readlines()
|
||||
if "torch" in "".join(lines).lower():
|
||||
with open(file, "w") as f:
|
||||
for line in lines:
|
||||
if (
|
||||
args.prefix
|
||||
and not line.lower().strip().startswith(TORCH_LIB_PREFIXES)
|
||||
or not args.prefix
|
||||
and "torch" not in line.lower()
|
||||
):
|
||||
f.write(line)
|
||||
else:
|
||||
print(f">>> removed from {file}:", line.strip())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
Reference in New Issue
Block a user