[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
+2 -2
View File
@@ -2,6 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Kernel implementations for vLLM."""
from . import aiter_ops, oink_ops, triton, vllm_c
from . import aiter_ops, triton, vllm_c
__all__ = ["vllm_c", "aiter_ops", "oink_ops", "triton"]
__all__ = ["vllm_c", "aiter_ops", "triton"]
-37
View File
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helion integration for vLLM."""
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import (
ConfigManager,
ConfigSet,
)
from vllm.kernels.helion.register import (
ConfigPicker,
ConfiguredHelionKernel,
HelionKernelWrapper,
get_kernel_by_name,
get_registered_kernels,
register_kernel,
vllm_helion_lib,
)
from vllm.kernels.helion.utils import canonicalize_gpu_name, get_canonical_gpu_name
__all__ = [
# Config management
"CaseKey",
"ConfigManager",
"ConfigSet",
# Kernel registration
"ConfigPicker",
"ConfiguredHelionKernel",
"HelionKernelWrapper",
"get_kernel_by_name",
"get_registered_kernels",
"register_kernel",
"vllm_helion_lib",
# Utilities
"canonicalize_gpu_name",
"get_canonical_gpu_name",
]
-70
View File
@@ -1,70 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Structured key for identifying kernel config/autotune/benchmark cases."""
from __future__ import annotations
import json
from typing import Any
class CaseKey(dict[str, Any]):
"""Immutable, hashable dict for identifying kernel cases.
Used as the key for config lookup, autotuning, benchmarking, and
input generation. Behaves like a read-only dict and can be used
as a dict key or in sets.
The canonical string form (``__str__``) is stable JSON with sorted
keys. Use ``CaseKey.default()`` for the default/fallback key.
The regular constructor requires at least one key-value pair::
CaseKey({"intermediate": 2048, "numtokens": 256})
CaseKey.default() # default/fallback
"""
def __init__(self, *args: Any, _allow_empty: bool = False, **kwargs: Any):
super().__init__(*args, **kwargs)
if not self and not _allow_empty:
raise TypeError(
"CaseKey requires at least one key-value pair. "
"Use CaseKey.default() for the default config key."
)
self._str: str | None = None
self._hash: int | None = None
@classmethod
def default(cls) -> CaseKey:
"""Create a default case key (empty)."""
return cls(_allow_empty=True)
def __hash__(self) -> int: # type: ignore[override]
if self._hash is None:
self._hash = hash(str(self))
return self._hash
def __str__(self) -> str:
if self._str is None:
self._str = json.dumps(dict(self), sort_keys=True, separators=(",", ":"))
return self._str
def __repr__(self) -> str:
if not self:
return "CaseKey.default()"
return f"CaseKey({dict(self)})"
def is_default(self) -> bool:
"""Return True if this is the default case key (empty)."""
return not self
def _readonly(self, *args: Any, **kwargs: Any) -> Any:
raise TypeError("CaseKey is immutable")
__setitem__ = _readonly # type: ignore[assignment]
__delitem__ = _readonly # type: ignore[assignment]
__ior__ = _readonly # type: ignore[assignment]
update = _readonly # type: ignore[assignment]
pop = _readonly # type: ignore[assignment]
popitem = _readonly # type: ignore[assignment]
setdefault = _readonly # type: ignore[assignment]
clear = _readonly # type: ignore[assignment]
-321
View File
@@ -1,321 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration management for Helion kernels.
This module provides centralized configuration file management for Helion custom
operations, including naming conventions, directory resolution, and file I/O.
Config File Structure
---------------------
Each kernel has a directory: {kernel_name}/
Inside, each GPU platform has its own JSON file: {kernel_name}/{platform}.json
Platform files store config entries as a JSON array::
[
{"key": {}, "config": {...}},
{"key": {"intermediate": 2048, "numtokens": 256}, "config": {...}},
...,
]
Config keys are ``CaseKey`` instances mapping parameter names to
values. The default config uses ``CaseKey.default()``.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"ConfigManager requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
logger = init_logger(__name__)
class ConfigSet:
"""In-memory collection of Helion configs with lookup/query capabilities.
Configs are stored keyed by ``CaseKey``. The default config
uses ``CaseKey.default()`` as its key.
"""
_ConfigDict = dict[str, dict[CaseKey, "helion.Config"]]
def __init__(self, kernel_name: str):
self._kernel_name = kernel_name
self._configs: ConfigSet._ConfigDict = {}
@property
def kernel_name(self) -> str:
return self._kernel_name
def get_config(self, platform: str, config_key: CaseKey) -> helion.Config:
platform_dict = self._configs.get(platform)
if platform_dict is None:
avail_platforms = self.get_platforms()
# TODO(@gmagogsfm): add a CLI/env override flag so users can
# directly specify a platform name instead of relying on
# auto-detection, and suggest it in this error message.
raise KeyError(
f"Config not found for kernel '{self._kernel_name}': "
f"platform '{platform}' not found. "
f"Available platforms: {avail_platforms or '(none)'}. "
f"If your GPU is a variant of a supported platform, "
f"consider adding a mapping in _GPU_NAME_ALIASES in "
f"vllm/kernels/helion/utils.py, or run "
f"tools/autotune_helion_kernels.py to generate configs "
f"for your platform."
)
config = platform_dict.get(config_key)
if config is None:
avail_keys = self.get_config_keys(platform)
raise KeyError(
f"Config not found for kernel '{self._kernel_name}': "
f"config_key '{config_key}' not found for "
f"platform '{platform}'. "
f"Available config_keys: {avail_keys or '(none)'}"
)
return config
def get_platforms(self) -> list[str]:
return sorted(self._configs.keys())
def get_config_keys(self, platform: str) -> list[CaseKey]:
platform_dict = self._configs.get(platform.lower())
if platform_dict is None:
return []
return sorted(platform_dict.keys(), key=str)
def to_config_entries(self) -> dict[str, list[dict[str, Any]]]:
"""Serialize to config entries format for JSON output."""
result: dict[str, list[dict[str, Any]]] = {}
for platform, config_dict in self._configs.items():
pairs: list[dict[str, Any]] = []
for config_key, config in config_dict.items():
config_data = json.loads(config.to_json())
pairs.append({"key": dict(config_key), "config": config_data})
result[platform] = pairs
return result
def to_dict(self) -> dict[str, dict[CaseKey, Any]]:
"""Return configs as a nested dict (platform -> key -> config)."""
result: dict[str, dict[CaseKey, Any]] = {}
for platform, config_dict in self._configs.items():
result[platform] = {
k: json.loads(v.to_json()) for k, v in config_dict.items()
}
return result
@classmethod
def from_dict(cls, kernel_name: str, data: dict[str, Any]) -> ConfigSet:
config_set = cls(kernel_name)
count = 0
for platform, platform_data in data.items():
if platform not in config_set._configs:
config_set._configs[platform] = {}
for entry in platform_data:
raw_key = entry["key"]
key = CaseKey.default() if not raw_key else CaseKey(raw_key)
config = helion.Config(**entry["config"])
config_set._configs[platform][key] = config
count += 1
if count > 0:
logger.debug(
"Loaded %d configs for kernel '%s'",
count,
kernel_name,
)
return config_set
def set_config(
self,
platform: str,
config_key: CaseKey,
config: helion.Config,
) -> None:
platform = platform.lower()
if platform not in self._configs:
self._configs[platform] = {}
self._configs[platform][config_key] = config
logger.debug(
"Set config for kernel '%s': platform='%s', key='%s'",
self._kernel_name,
platform,
config_key,
)
class ConfigManager:
"""File-level configuration management for Helion kernels (global singleton)."""
_instance: ConfigManager | None = None
_instance_base_dir: Path | None = None
def __new__(cls, base_dir: str | Path | None = None) -> ConfigManager:
resolved_base_dir = cls._resolve_base_dir(base_dir)
if cls._instance is not None:
if cls._instance_base_dir != resolved_base_dir:
raise ValueError(
f"ConfigManager singleton already exists with base_dir "
f"'{cls._instance_base_dir}', cannot create with "
f"different base_dir '{resolved_base_dir}'"
)
return cls._instance
instance = super().__new__(cls)
cls._instance = instance
cls._instance_base_dir = resolved_base_dir
return instance
def __init__(self, base_dir: str | Path | None = None):
if hasattr(self, "_base_dir"):
return
self._base_dir = self._resolve_base_dir(base_dir)
logger.debug("ConfigManager initialized with base_dir: %s", self._base_dir)
@staticmethod
def _resolve_base_dir(base_dir: str | Path | None) -> Path:
if base_dir is not None:
return Path(base_dir).resolve()
return (Path(__file__).parent / "configs").resolve()
@classmethod
def get_instance(cls) -> ConfigManager:
if cls._instance is None:
raise RuntimeError(
"ConfigManager instance has not been created. "
"Call ConfigManager(base_dir=...) first to initialize."
)
return cls._instance
@classmethod
def reset_instance(cls) -> None:
"""For testing purposes only."""
cls._instance = None
cls._instance_base_dir = None
def get_kernel_dir(self, kernel_name: str) -> Path:
return self._base_dir / kernel_name
def get_config_file_path(
self, kernel_name: str, platform: str | None = None
) -> Path:
if platform is not None:
return self.get_kernel_dir(kernel_name) / f"{platform}.json"
return self.get_kernel_dir(kernel_name)
def ensure_base_dir_exists(self) -> Path:
self._base_dir.mkdir(parents=True, exist_ok=True)
return self._base_dir
def ensure_base_dir_writable(self) -> None:
self.ensure_base_dir_exists()
test_file = self._base_dir / ".write_test"
try:
test_file.write_text("test")
test_file.unlink()
except OSError as e:
raise OSError(
f"Config directory '{self._base_dir}' is not writable: {e}"
) from e
def _load_platform_file(self, kernel_name: str, platform: str) -> Any:
config_path = self.get_config_file_path(kernel_name, platform)
if not config_path.exists():
return []
try:
with open(config_path) as f:
return json.load(f)
except (json.JSONDecodeError, OSError) as e:
logger.error("Failed to load config file %s: %s", config_path, e)
return []
def load_config_set(self, kernel_name: str) -> ConfigSet:
kernel_dir = self.get_kernel_dir(kernel_name)
if not kernel_dir.is_dir():
return ConfigSet.from_dict(kernel_name, {})
data: dict[str, Any] = {}
for platform_file in sorted(kernel_dir.glob("*.json")):
platform = platform_file.stem
try:
with open(platform_file) as f:
platform_data = json.load(f)
data[platform] = platform_data
except (json.JSONDecodeError, OSError) as e:
logger.error(
"Failed to load config file %s: %s",
platform_file,
e,
)
return ConfigSet.from_dict(kernel_name, data)
def get_platform_configs(
self, kernel_name: str, platform: str
) -> dict[CaseKey, helion.Config]:
platform_data = self._load_platform_file(kernel_name, platform)
if not platform_data:
return {}
config_set = ConfigSet.from_dict(kernel_name, {platform: platform_data})
return {
k: config_set.get_config(platform, k)
for k in config_set.get_config_keys(platform)
}
def save_config_set(self, config_set: ConfigSet) -> Path:
kernel_dir = self.get_kernel_dir(config_set.kernel_name)
kernel_dir.mkdir(parents=True, exist_ok=True)
full_data = config_set.to_config_entries()
for platform, pairs in full_data.items():
platform_path = kernel_dir / f"{platform}.json"
with open(platform_path, "w") as f:
json.dump(pairs, f, indent=2)
f.write("\n")
logger.info("Saved config to: %s", platform_path)
return kernel_dir
def save_configs(
self,
kernel_name: str,
platform: str,
configs: dict[CaseKey, helion.Config],
) -> Path:
"""Save configs for a kernel/platform, merging with existing."""
config_set = ConfigSet.from_dict(
kernel_name,
{platform: self._load_platform_file(kernel_name, platform)},
)
for key, config in configs.items():
config_set.set_config(platform, key, config)
pairs = config_set.to_config_entries().get(platform, [])
platform_path = self.get_config_file_path(kernel_name, platform)
platform_path.parent.mkdir(parents=True, exist_ok=True)
with open(platform_path, "w") as f:
json.dump(pairs, f, indent=2)
f.write("\n")
logger.info("Saved config to: %s", platform_path)
return platform_path
-34
View File
@@ -1,34 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helion kernel implementation.
Importing this package does NOT register any kernels. Runtime code imports the
specific op module it needs, e.g.::
from vllm.kernels.helion.ops import scaled_mm # noqa: F401
which triggers that op's ``@register_kernel`` as an import side effect.
Tools that need the full registry (e.g. tools/autotune_helion_kernels.py)
call ``import_all_ops()`` to force every op module to register.
"""
import importlib
import pkgutil
def import_all_kernels() -> list[str]:
"""Import every kernel submodule so all ``@register_kernel`` decorators run.
Returns:
The fully-qualified module names that were imported.
"""
imported: list[str] = []
for module_info in pkgutil.iter_modules(__path__):
if module_info.ispkg:
continue
module_name = f"{__name__}.{module_info.name}"
importlib.import_module(module_name)
imported.append(module_name)
return imported
@@ -1,171 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.register import register_kernel
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
logger = init_logger(__name__)
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all input
# property combination. Currently, dtypes are fixed. We need optimization to
# bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
hidden_size_list = [2048, 4096, 5120]
in_dtype: torch.dtype = torch.bfloat16
out_dtype: torch.dtype = current_platform.fp8_dtype()
scale_dtype: torch.dtype = torch.float32
inputs = {}
for num_tokens, hidden_size in product(num_tokens_list, hidden_size_list):
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
result = torch.empty(input.shape, device=input.device, dtype=out_dtype)
scale = torch.empty((num_tokens, 1), device=input.device, dtype=scale_dtype)
# scale_ub clamps the per-token amax of |input|. Use a non-degenerate
# upper bound (midway between the mean and max of |input|) so clamping is
# partially active and the baseline comparison is meaningful.
# torch.mean(input) ~= 0 for the zero-mean input would collapse every
# scale to the floor and saturate the output.
input_abs = input.to(torch.float32).abs()
scale_ub = (0.5 * (input_abs.mean() + input_abs.amax())).to(scale_dtype)
config_key = CaseKey({"hidden_size": hidden_size, "num_tokens": num_tokens})
inputs[config_key] = (result, input, scale, scale_ub)
return inputs
_pick_cache: dict[tuple[int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest hidden_size among available configs
(exact match preferred).
2. Among the num_tokens values tuned for that hidden_size, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
_, input, *_ = args
num_tokens, hidden_size = input.shape
cache_key = (num_tokens, hidden_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, list[int]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["hidden_size"], []).append(key["num_tokens"])
if not configs:
return None
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
available_num_tokens = sorted(configs[best_hidden_size])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey({"hidden_size": best_hidden_size, "num_tokens": best_num_tokens})
_pick_cache[cache_key] = result
return result
def baseline(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
scale: torch.Tensor, # [num_tokens, 1]
scale_ub: torch.Tensor | None = None, # scalar tensor
) -> None:
fp8_min, fp8_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (fp8_max * 512.0)
x = input.to(torch.float32)
s = torch.amax(torch.abs(x), dim=-1, keepdim=True)
if scale_ub is not None:
s = s.clamp(max=scale_ub)
s = (s * (1.0 / fp8_max)).clamp(min=min_scaling_factor)
y = (x / s).clamp(fp8_min, fp8_max)
scale.copy_(s)
result.copy_(y.to(result.dtype))
# Overwrite autotune_baseline_atol and autotune_baseline_rtol
# if too many configs failed due to baseline check during autotuning
@register_kernel(
mutates_args=["result", "scale"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
ignore_warnings=[helion.exc.TensorOperationInWrapper],
),
)
def dynamic_per_token_scaled_fp8_quant(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
scale: torch.Tensor, # [num_tokens, 1]
scale_ub: torch.Tensor | None = None, # scalar tensor
) -> None:
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
num_tokens, hidden_size = input.shape
hl.specialize(hidden_size)
assert result.shape == input.shape
assert scale.shape[0] == num_tokens
assert scale.dtype == torch.float32
assert input.stride()[-1] == 1
assert result.stride()[-1] == 1
fp8_min, fp8_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (fp8_max * 512.0)
for tile_m in hl.tile(num_tokens, block_size=1):
s_blk = hl.zeros([tile_m], dtype=torch.float32)
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(dtype=torch.float32)
tmp_blk = torch.amax(torch.abs(x_blk), dim=-1)
s_blk = torch.maximum(s_blk, tmp_blk)
if scale_ub is not None:
scale_ub_s = hl.load(scale_ub, [])
s_blk = s_blk.clamp(max=scale_ub_s)
s_blk = s_blk * (1.0 / fp8_max)
s_blk = s_blk.clamp(min=min_scaling_factor)
scale[tile_m, 0] = s_blk
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(torch.float32)
y_blk = x_blk * (1.0 / s_blk[:, None])
result[tile_m, tile_n] = y_blk.clamp(fp8_min, fp8_max).to(result.dtype)
@@ -1,297 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
from vllm import ir
from vllm.kernels.helion.register import register_kernel
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
logger = init_logger(__name__)
def _compute_cos_sin_cache(
max_position_embeddings, rotary_dim, device="cuda", dtype=torch.float
):
inv_freq = 1.0 / (
10000
** (torch.arange(0, rotary_dim, 2, device=device, dtype=dtype) / rotary_dim)
)
t = torch.arange(max_position_embeddings, device=device, dtype=dtype)
freqs = torch.einsum("i,j -> ij", t, inv_freq)
cos = freqs.cos()
sin = freqs.sin()
cache = torch.cat((cos, sin), dim=-1)
return cache
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover
# all input property combination. Currently, dtypes are fixed. We need
# optimization to bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
num_heads_pair = [
(16, 8),
(32, 8),
(64, 8),
]
head_dim = 128
in_dtype: torch.dtype = torch.bfloat16
rotary_ratio = 1.0
is_neox = True
eps = 1e-6
device = "cuda"
inputs = {}
for num_tokens, (num_q_heads, num_kv_heads) in product(
num_tokens_list, num_heads_pair
):
total_dim = (num_q_heads + 2 * num_kv_heads) * head_dim
qkv = torch.empty(
num_tokens, total_dim, dtype=in_dtype, device=device
).uniform_(-0.1, 0.1)
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
q_weight = torch.empty(head_dim, dtype=in_dtype, device=device).uniform_(
0.8, 1.2
)
k_weight = torch.empty(head_dim, dtype=in_dtype, device=device).uniform_(
0.8, 1.2
)
rotary_dim = int(head_dim * rotary_ratio)
cos_sin_cache = _compute_cos_sin_cache(40960, rotary_dim)
cos_sin_cache = cos_sin_cache.to(in_dtype)
config_key = CaseKey(
{
"q_heads": num_q_heads,
"kv_heads": num_kv_heads,
"num_tokens": num_tokens,
}
)
inputs[config_key] = (
qkv,
num_q_heads,
num_kv_heads,
num_kv_heads,
head_dim,
eps,
q_weight,
k_weight,
cos_sin_cache,
is_neox,
positions.view(-1),
)
return inputs
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest q_heads among available configs
(exact match preferred).
2. Find the closest kv_heads among available configs
(exact match preferred).
3. Among the num_tokens values tuned for that q_heads and q_heads, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
qkv, q_heads, kv_heads, *_ = args
num_tokens = qkv.shape[0]
cache_key = (num_tokens, q_heads, kv_heads)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, dict[int, list[int]]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["q_heads"], {}).setdefault(key["kv_heads"], []).append(
key["num_tokens"]
)
if not configs:
return None
best_q_heads = min(configs, key=lambda s: abs(s - q_heads))
best_kv_heads = min(configs[best_q_heads], key=lambda s: abs(s - kv_heads))
available_num_tokens = sorted(configs[best_q_heads][best_kv_heads])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey(
{
"q_heads": best_q_heads,
"kv_heads": best_kv_heads,
"num_tokens": best_num_tokens,
}
)
_pick_cache[cache_key] = result
return result
def baseline(
qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]
num_heads_q: int,
num_heads_k: int,
num_heads_v: int,
head_dim: int,
eps: float,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor, # [max_position, rotary_dim]
is_neox: bool,
position_ids: torch.Tensor, # [num_tokens],
forced_token_heads_per_warp: int = -1, # dummy
) -> None:
q_size = num_heads_q * head_dim
kv_size = num_heads_k * head_dim
q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
q_by_head = q.view(*q.shape[:-1], q.shape[-1] // head_dim, head_dim)
q_by_head = ir.ops.rms_norm(q_by_head, q_weight, eps)
q = q_by_head.view(q.shape)
k_by_head = k.view(*k.shape[:-1], k.shape[-1] // head_dim, head_dim)
k_by_head = ir.ops.rms_norm(k_by_head, k_weight, eps)
k = k_by_head.view(k.shape)
q, k = RotaryEmbedding.forward_static(
position_ids, q, k, head_dim, cos_sin_cache.shape[1], cos_sin_cache, is_neox
)
qkv[:, :q_size].copy_(q)
qkv[:, q_size : q_size + kv_size].copy_(k)
# Overwrite autotune_baseline_atol and autotune_baseline_rtol
# if too many configs failed due to baseline check during autotuning
@register_kernel(
mutates_args=["qkv"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
autotune_baseline_atol=5e-2,
autotune_baseline_rtol=5e-2,
ignore_warnings=[helion.exc.TensorOperationInWrapper],
),
) # type: ignore[misc]
def fused_qk_norm_rope(
qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]
num_heads_q: int,
num_heads_k: int,
num_heads_v: int,
head_dim: int,
eps: float,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor, # [max_position, rotary_dim]
is_neox: bool,
position_ids: torch.Tensor, # [num_tokens],
forced_token_heads_per_warp: int = -1, # dummy
) -> None:
assert qkv.ndim == 2
num_tokens = qkv.shape[0]
total_heads = num_heads_q + num_heads_k + num_heads_v
assert qkv.shape[1] == total_heads * head_dim
hl.specialize(qkv.shape[1])
assert cos_sin_cache.ndim == 2
max_position, rotary_dim = cos_sin_cache.shape
hl.specialize(max_position)
hl.specialize(rotary_dim)
assert rotary_dim % 2 == 0
assert rotary_dim <= head_dim
embed_dim = rotary_dim // 2
hl.specialize(num_heads_q)
hl.specialize(num_heads_k)
hl.specialize(num_heads_v)
hl.specialize(head_dim)
assert position_ids.ndim == 1 and position_ids.shape[0] == num_tokens
hl.specialize(position_ids.shape[0])
assert q_weight.ndim == 1 and q_weight.shape[0] == head_dim
hl.specialize(q_weight.shape[0])
assert k_weight.ndim == 1 and k_weight.shape[0] == head_dim
hl.specialize(k_weight.shape[0])
assert qkv.dtype == q_weight.dtype and q_weight.dtype == k_weight.dtype
assert position_ids.dtype == torch.int64
assert qkv.is_contiguous()
assert position_ids.is_contiguous()
assert q_weight.is_contiguous()
assert k_weight.is_contiguous()
assert cos_sin_cache.is_contiguous()
qk_heads = num_heads_q + num_heads_k
qkv = qkv.view(num_tokens, -1, head_dim)
for tile_m, tile_gn, tile_n in hl.tile(
[num_tokens, qk_heads, head_dim], block_size=[1, None, head_dim]
):
x_blk = qkv[tile_m, tile_gn, tile_n].to(dtype=torch.float32)
rms = x_blk.pow(2).sum(dim=-1)
rms = torch.rsqrt(rms * (1.0 / head_dim) + eps)
use_q_weight = (tile_gn.index < num_heads_q)[None, :, None]
w_blk = torch.where(
use_q_weight, q_weight[None, None, tile_n], k_weight[None, None, tile_n]
)
x_blk = (x_blk * rms[:, :, None]).to(qkv.dtype) * w_blk
qkv[tile_m, tile_gn, tile_n] = x_blk
pos_id = position_ids[tile_m]
cos_blk = cos_sin_cache[pos_id, hl.arange(embed_dim)]
sin_blk = cos_sin_cache[pos_id, hl.arange(embed_dim) + embed_dim]
if is_neox:
x1_offset = hl.arange(embed_dim)
x2_offset = x1_offset + embed_dim
else:
x1_offset = hl.arange(embed_dim) * 2
x2_offset = x1_offset + 1
x1_blk = qkv[tile_m, tile_gn, x1_offset]
x2_blk = qkv[tile_m, tile_gn, x2_offset]
o1_blk = x1_blk * cos_blk[:, None, :] - x2_blk * sin_blk[:, None, :]
o2_blk = x2_blk * cos_blk[:, None, :] + x1_blk * sin_blk[:, None, :]
qkv[tile_m, tile_gn, x1_offset] = o1_blk
qkv[tile_m, tile_gn, x2_offset] = o2_blk
@@ -1,212 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
from vllm.kernels.helion.register import register_kernel
logger = init_logger(__name__)
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all
# input property combination. Currently, dtypes are fixed. We need
# optimization to bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
hidden_size_list = [2048, 4096, 5120]
group_size_list = [128]
in_dtype: torch.dtype = torch.bfloat16
out_dtype: torch.dtype = current_platform.fp8_dtype()
scale_dtype: torch.dtype = torch.float32
use_ue8m0 = False
column_major = False
fp8_min, fp8_max = get_fp8_min_max()
eps = 1e-10
inputs = {}
for hidden_size, group_size, num_tokens in product(
hidden_size_list, group_size_list, num_tokens_list
):
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
output_q = torch.empty(input.shape, device=input.device, dtype=out_dtype)
output_s = torch.empty(
(num_tokens, hidden_size // group_size),
device=input.device,
dtype=scale_dtype,
)
config_key = CaseKey(
{
"hidden_size": hidden_size,
"group_size": group_size,
"num_tokens": num_tokens,
}
)
inputs[config_key] = (
input,
output_q,
output_s,
group_size,
eps,
fp8_min,
fp8_max,
use_ue8m0,
column_major,
False,
)
return inputs
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest hidden_size among available configs
(exact match preferred).
2. Find the closest group_size among available configs
(exact match preferred).
3. Among the num_tokens values tuned for that hidden_size and group_size, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
input, _, _, group_size, *_ = args
num_tokens, hidden_size = input.shape
cache_key = (num_tokens, group_size, hidden_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, dict[int, list[int]]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["hidden_size"], {}).setdefault(
key["group_size"], []
).append(key["num_tokens"])
if not configs:
return None
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
best_group_size = min(configs[best_hidden_size], key=lambda s: abs(s - group_size))
available_num_tokens = sorted(configs[best_hidden_size][best_group_size])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey(
{
"hidden_size": best_hidden_size,
"group_size": best_group_size,
"num_tokens": best_num_tokens,
}
)
_pick_cache[cache_key] = result
return result
def baseline(
input: torch.Tensor, # [num_tokens, hidden_size]
output_q: torch.Tensor, # [num_tokens, hidden_size]
output_s: torch.Tensor, # [num_tokens, groups_per_row]
group_size: int,
eps: float,
fp8_min: float,
fp8_max: float,
scale_ue8m0: bool,
dummy_is_scale_transposed: bool = False,
dummy_is_tma_aligned: bool = False,
) -> None:
num_tokens, hidden_size = input.shape
groups_per_row = hidden_size // group_size
x = input.view(num_tokens, groups_per_row, group_size).to(torch.float32)
s = torch.clamp(torch.amax(torch.abs(x), dim=-1), min=eps) / fp8_max
if scale_ue8m0:
s = torch.exp2(torch.ceil(torch.log2(s)))
y = torch.clamp(x / s[:, :, None], fp8_min, fp8_max)
output_s.copy_(s)
output_q.copy_(y.view(num_tokens, hidden_size).to(output_q.dtype))
@register_kernel(
mutates_args=["output_q", "output_s"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
),
) # type: ignore[misc]
def per_token_group_fp8_quant(
input: torch.Tensor, # [num_tokens, hidden_size]
output_q: torch.Tensor, # [num_tokens, hidden_size]
output_s: torch.Tensor, # [num_tokens, groups_per_row]
group_size: int,
eps: float,
fp8_min: float,
fp8_max: float,
scale_ue8m0: bool,
# Unused dummy args
# Kept for consistency with existing kernel interface
dummy_is_scale_transposed: bool = False,
dummy_is_tma_aligned: bool = False,
) -> None:
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
num_tokens, hidden_size = input.shape
hl.specialize(hidden_size)
hl.specialize(group_size)
groups_per_row = output_s.shape[1]
hl.specialize(groups_per_row)
assert hidden_size % group_size == 0 and hidden_size // group_size == groups_per_row
assert output_s.ndim == 2 and output_s.dtype == torch.float32
input = input.view(num_tokens, -1, group_size)
output_q = output_q.view(num_tokens, -1, group_size)
for tile_m, tile_gn, tile_n in hl.tile(
[num_tokens, groups_per_row, group_size], block_size=[1, None, group_size]
):
x_blk = input[tile_m, tile_gn, tile_n]
y_s_blk = torch.clamp(torch.amax(torch.abs(x_blk), dim=-1), min=eps)
y_s_blk = y_s_blk / fp8_max
if scale_ue8m0:
y_s_blk = torch.exp2(torch.ceil(torch.log2(y_s_blk)))
y_q_blk = torch.clamp(x_blk / y_s_blk[:, :, None], fp8_min, fp8_max).to(
output_q.dtype
)
output_s[tile_m, tile_gn] = y_s_blk
output_q[tile_m, tile_gn, tile_n] = y_q_blk
@@ -1,256 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.utils import (
get_fp8_dtype,
get_int8_min_max,
get_int8_min_scaling_factor,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
from vllm.kernels.helion.register import register_kernel
logger = init_logger(__name__)
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all
# input property combination. Currently, dtypes are fixed. We need
# optimization to bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
hidden_size_list = [2048, 4096, 5120]
in_dtype: torch.dtype = torch.bfloat16
out_dtype: torch.dtype = current_platform.fp8_dtype()
scale_dtype: torch.dtype = torch.float32
inputs = {}
for num_tokens, hidden_size in product(num_tokens_list, hidden_size_list):
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
result = torch.empty(input.shape, device=input.device, dtype=out_dtype)
scale = torch.empty((num_tokens, 1), device=input.device, dtype=scale_dtype)
residual = torch.randn_like(input)
weight = torch.normal(
mean=1.0,
std=1.0,
size=(hidden_size,),
dtype=input.dtype,
device=input.device,
)
epsilon = 1e-6
# scale_ub clamps the per-token amax of the RMS-normed, weighted output.
# Use a non-degenerate upper bound (midway between the mean and max of
# that magnitude) so clamping is partially active and the baseline
# comparison is meaningful. torch.mean(input) ~= 0 for the zero-mean
# input would collapse every scale to the floor and saturate the output.
# Mirrors the reference normalization in baseline() below.
x = input.to(torch.float32) + residual.to(torch.float32)
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
x_norm_abs = ((x * rms).to(input.dtype) * weight).abs().to(torch.float32)
scale_ub = (0.5 * (x_norm_abs.mean() + x_norm_abs.amax())).to(scale_dtype)
config_key = CaseKey({"hidden_size": hidden_size, "num_tokens": num_tokens})
inputs[config_key] = (result, input, weight, scale, epsilon, scale_ub, residual)
return inputs
_pick_cache: dict[tuple[int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest hidden_size among available configs
(exact match preferred).
2. Among the num_tokens values tuned for that hidden_size, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
_, input, *_ = args
num_tokens, hidden_size = input.shape
cache_key = (num_tokens, hidden_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, list[int]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["hidden_size"], []).append(key["num_tokens"])
if not configs:
return None
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
available_num_tokens = sorted(configs[best_hidden_size])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey({"hidden_size": best_hidden_size, "num_tokens": best_num_tokens})
_pick_cache[cache_key] = result
return result
def baseline(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
weight: torch.Tensor, # [num_tokens]
scale: torch.Tensor, # [num_tokens, 1]
epsilon: float,
scale_ub: torch.Tensor | None = None, # []
residual: torch.Tensor | None = None, # [num_tokens, hidden_size]
) -> None:
_, hidden_size = input.shape
quant_dtype = result.dtype
qtype_min: int | float
qtype_max: int | float
if quant_dtype == torch.int8:
qtype_min, qtype_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_min, qtype_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_max * 512.0)
x = input.to(torch.float32)
if residual is not None:
x = x + residual
residual.copy_(x.to(residual.dtype))
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
x_norm = (x * rms).to(input.dtype) * weight
s = torch.amax(torch.abs(x_norm), dim=-1, keepdim=True).to(torch.float32)
if scale_ub is not None:
s = s.clamp(max=scale_ub)
s = (s * (1.0 / qtype_max)).clamp(min=min_scaling_factor)
y = x_norm / s
if quant_dtype == torch.int8:
y = y.round()
scale.copy_(s)
result.copy_(y.clamp(qtype_min, qtype_max).to(result.dtype))
# Overwrite autotune_baseline_atol and autotune_baseline_rtol
# if too many configs failed due to baseline check during autotuning
@register_kernel(
mutates_args=["result", "scale", "residual"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
ignore_warnings=[helion.exc.TensorOperationInWrapper],
),
) # type: ignore[misc]
def rms_norm_dynamic_per_token_quant(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
weight: torch.Tensor, # [hidden_size]
scale: torch.Tensor, # [num_tokens, 1]
epsilon: float,
scale_ub: torch.Tensor | None = None, # []
residual: torch.Tensor | None = None, # [num_tokens, hidden_size]
) -> None:
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
num_tokens, hidden_size = input.shape
hl.specialize(hidden_size)
fp8_dtype = get_fp8_dtype()
assert result.dtype in [fp8_dtype, torch.int8]
assert result.is_contiguous() and input.is_contiguous()
if scale_ub is not None:
assert result.dtype == fp8_dtype
assert scale_ub.dtype == torch.float32
assert input.dtype == weight.dtype
assert scale.shape[0] == num_tokens
assert scale.dtype == torch.float32
if residual is not None:
assert residual.dtype == input.dtype
quant_dtype = result.dtype
qtype_traits_min: int | float
qtype_traits_max: int | float
if quant_dtype == torch.int8:
qtype_traits_min, qtype_traits_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_traits_max * 512.0)
qtype_max = float(qtype_traits_max)
for tile_m in hl.tile(num_tokens, block_size=1):
rms = hl.zeros([tile_m], dtype=torch.float32)
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(torch.float32)
if residual is not None:
x_blk = x_blk + residual[tile_m, tile_n]
rms = rms + x_blk.pow(2).sum(dim=-1)
rms = torch.rsqrt(rms * (1.0 / hidden_size) + epsilon)
s_blk = hl.zeros([tile_m], dtype=torch.float32)
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(torch.float32)
if residual is not None:
x_blk = x_blk + residual[tile_m, tile_n]
x_blk = (x_blk * rms[:, None]).to(input.dtype) * weight[None, tile_n]
tmp_blk = torch.amax(torch.abs(x_blk), dim=-1).to(torch.float32)
s_blk = torch.maximum(s_blk, tmp_blk)
if scale_ub is not None:
scale_ub_s = hl.load(scale_ub, [])
s_blk = s_blk.clamp(max=scale_ub_s)
s_blk = s_blk * (1.0 / qtype_max)
s_blk = s_blk.clamp(min=min_scaling_factor)
scale[tile_m, 0] = s_blk
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(torch.float32)
if residual is not None:
x_blk = x_blk + residual[tile_m, tile_n]
residual[tile_m, tile_n] = x_blk.to(residual.dtype)
x_blk = (x_blk * rms[:, None]).to(input.dtype) * weight[None, tile_n]
if quant_dtype == torch.int8:
s_inv_blk = 1.0 / s_blk[:, None]
y_blk = x_blk * s_inv_blk
y_blk = y_blk.round()
else:
y_blk = x_blk / s_blk[:, None]
result[tile_m, tile_n] = y_blk.clamp(qtype_traits_min, qtype_traits_max).to(
result.dtype
)
@@ -1,316 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.utils import (
get_fp8_dtype,
get_int8_min_max,
get_int8_min_scaling_factor,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
from vllm.kernels.helion.register import register_kernel
logger = init_logger(__name__)
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all
# input property combination. Currently, dtypes are fixed. We need
# optimization to bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
hidden_size_list = [2048, 4096, 5120]
group_size_list = [128]
in_dtype: torch.dtype = torch.bfloat16
out_dtype: torch.dtype = current_platform.fp8_dtype()
scale_dtype: torch.dtype = torch.float32
inputs = {}
for hidden_size, group_size, num_tokens in product(
hidden_size_list, group_size_list, num_tokens_list
):
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
result = torch.empty(input.shape, device=input.device, dtype=out_dtype)
scale = torch.empty(
(num_tokens, hidden_size // group_size),
device=input.device,
dtype=scale_dtype,
)
residual = torch.randn_like(input)
weight = torch.normal(
mean=1.0,
std=1.0,
size=(hidden_size,),
dtype=input.dtype,
device=input.device,
)
epsilon = 1e-6
# scale_ub clamps the per-group amax of the RMS-normed, weighted output.
# Use a non-degenerate upper bound (midway between the mean and max of
# that magnitude) so clamping is partially active and the baseline
# comparison is meaningful. torch.mean(input) ~= 0 for the zero-mean
# input would collapse every scale to the floor and saturate the output.
# Mirrors the reference normalization in baseline() below.
x = input.to(torch.float32) + residual.to(torch.float32)
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
x_norm_abs = ((x * rms).to(input.dtype) * weight).abs().to(torch.float32)
scale_ub = (0.5 * (x_norm_abs.mean() + x_norm_abs.amax())).to(scale_dtype)
config_key = CaseKey(
{
"hidden_size": hidden_size,
"group_size": group_size,
"num_tokens": num_tokens,
}
)
inputs[config_key] = (
result,
input,
weight,
scale,
epsilon,
scale_ub,
residual,
group_size,
False,
)
return inputs
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest hidden_size among available configs
(exact match preferred).
2. Find the closest group_size among available configs
(exact match preferred).
2. Among the num_tokens values tuned for that hidden_size and group_size, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
_, input, _, _, _, _, _, group_size, *_ = args
num_tokens, hidden_size = input.shape
cache_key = (num_tokens, group_size, hidden_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, dict[int, list[int]]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["hidden_size"], {}).setdefault(
key["group_size"], []
).append(key["num_tokens"])
if not configs:
return None
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
best_group_size = min(configs[best_hidden_size], key=lambda s: abs(s - group_size))
available_num_tokens = sorted(configs[best_hidden_size][best_group_size])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey(
{
"hidden_size": best_hidden_size,
"group_size": best_group_size,
"num_tokens": best_num_tokens,
}
)
_pick_cache[cache_key] = result
return result
def baseline(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
weight: torch.Tensor, # [hidden_size]
scale: torch.Tensor, # [num_tokens, groups_per_row]
epsilon: float,
scale_ub: torch.Tensor | None, # []
residual: torch.Tensor | None, # [num_tokens, hidden_size]
group_size: int,
is_scale_transposed: bool,
) -> None:
num_tokens, hidden_size = input.shape
groups_per_row = hidden_size // group_size
quant_dtype = result.dtype
qtype_min: int | float
qtype_max: int | float
if quant_dtype == torch.int8:
qtype_min, qtype_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_min, qtype_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_max * 512.0)
x = input.to(torch.float32)
if residual is not None:
x = x + residual
residual.copy_(x.to(residual.dtype))
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
x_norm = (x * rms).to(input.dtype) * weight
x_grouped = x_norm.view(num_tokens, groups_per_row, group_size).to(torch.float32)
s = torch.amax(torch.abs(x_grouped), dim=-1).to(torch.float32)
if scale_ub is not None:
s = s.clamp(max=scale_ub)
s = (s * (1.0 / qtype_max)).clamp(min=min_scaling_factor)
y = x_grouped / s[:, :, None]
if quant_dtype == torch.int8:
y = y.round()
scale.copy_(s)
result.copy_(
y.clamp(qtype_min, qtype_max).view(num_tokens, hidden_size).to(result.dtype)
)
@register_kernel(
mutates_args=["result", "scale", "residual"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
ignore_warnings=[helion.exc.TensorOperationInWrapper],
),
) # type: ignore[misc]
def rms_norm_per_block_quant(
result: torch.Tensor, # [num_tokens, hidden_size]
input: torch.Tensor, # [num_tokens, hidden_size]
weight: torch.Tensor, # [hidden_size]
scale: torch.Tensor, # [num_tokens, groups_per_row]
epsilon: float,
scale_ub: torch.Tensor | None, # []
residual: torch.Tensor | None, # [num_tokens, hidden_size]
group_size: int,
is_scale_transposed: bool, # dummy
) -> None:
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
num_tokens, hidden_size = input.shape
hl.specialize(hidden_size)
hl.specialize(group_size)
groups_per_row = scale.shape[1]
hl.specialize(groups_per_row)
assert hidden_size % group_size == 0 and hidden_size // group_size == groups_per_row
assert scale.shape[0] == num_tokens
assert scale.dtype == torch.float32
if scale.stride(1) > 1:
assert is_scale_transposed
fp8_dtype = get_fp8_dtype()
assert result.dtype in [fp8_dtype, torch.int8]
assert result.is_contiguous() and input.is_contiguous()
if scale_ub is not None:
assert result.dtype == fp8_dtype
assert scale_ub.dtype == torch.float32
assert input.dtype == weight.dtype
if residual is not None:
assert residual.dtype == input.dtype
assert group_size in [64, 128]
quant_dtype = result.dtype
qtype_traits_min: int | float
qtype_traits_max: int | float
if quant_dtype == torch.int8:
qtype_traits_min, qtype_traits_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_traits_max * 512.0)
qtype_max = float(qtype_traits_max)
for tile_m in hl.tile(num_tokens, block_size=1):
rms = hl.zeros([tile_m], dtype=torch.float32)
for tile_n in hl.tile(hidden_size):
x_blk = input[tile_m, tile_n].to(torch.float32)
if residual is not None:
x_blk = x_blk + residual[tile_m, tile_n]
rms = rms + x_blk.pow(2).sum(dim=-1)
rms = torch.rsqrt(rms * (1.0 / hidden_size) + epsilon)
m_idx = tile_m.begin + hl.arange(tile_m.block_size)
m_blk = m_idx[:, None, None]
for tile_gn, tile_n in hl.tile(
[groups_per_row, group_size], block_size=[None, group_size]
):
gn_idx = tile_gn.index
n_offset = tile_n.index
n_idx = gn_idx[:, None] * group_size + n_offset[None, :]
n_blk = n_idx[None, :, :]
mask = (gn_idx < groups_per_row)[None, :, None]
x_blk = hl.load(input, [m_blk, n_blk], extra_mask=mask).to(
dtype=torch.float32
)
if residual is not None:
r_blk = hl.load(residual, [m_blk, n_blk], extra_mask=mask)
x_blk = x_blk + r_blk
w_blk = hl.load(weight, [n_blk], extra_mask=mask)
x_norm_blk = (x_blk * rms[:, None, None]).to(input.dtype) * w_blk
s_blk = torch.amax(torch.abs(x_norm_blk), dim=-1).to(torch.float32)
if scale_ub is not None:
scale_ub_s = hl.load(scale_ub, [])
s_blk = s_blk.clamp(max=scale_ub_s)
s_blk = s_blk * (1.0 / qtype_max)
s_blk = s_blk.clamp(min=min_scaling_factor)
scale[tile_m, tile_gn] = s_blk
if quant_dtype == torch.int8:
y_blk = (x_norm_blk * (1.0 / s_blk[:, :, None])).round()
else:
y_blk = x_norm_blk / s_blk[:, :, None]
y_blk = y_blk.clamp(qtype_traits_min, qtype_traits_max).to(result.dtype)
hl.store(result, [m_blk, n_blk], y_blk, extra_mask=mask)
if residual is not None:
hl.store(
residual, [m_blk, n_blk], x_blk.to(residual.dtype), extra_mask=mask
)
@@ -1,273 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.utils import (
get_fp8_dtype,
get_int8_min_max,
get_int8_min_scaling_factor,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
from vllm.kernels.helion.register import register_kernel
logger = init_logger(__name__)
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all input
# property combination. Currently, dtypes are fixed. We need optimization to
# bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
intermediate_size_list = [6144, 12288, 25600]
in_dtype: torch.dtype = torch.bfloat16
out_dtype: torch.dtype = current_platform.fp8_dtype()
scale_dtype: torch.dtype = torch.float32
group_size_list = [128]
inputs = {}
for intermediate_size, group_size, num_tokens in product(
intermediate_size_list, group_size_list, num_tokens_list
):
input = torch.randn(
num_tokens, 2 * intermediate_size, device="cuda", dtype=in_dtype
)
result = torch.empty(
num_tokens, intermediate_size, device=input.device, dtype=out_dtype
)
scale = torch.empty(
(num_tokens, intermediate_size // group_size),
device=input.device,
dtype=scale_dtype,
)
# scale_ub clamps the per-group amax of the SiLU-and-mul activation. Use
# a non-degenerate upper bound (midway between the mean and max of the
# activation magnitude) so clamping is partially active and the baseline
# comparison is meaningful. torch.mean(input) ~= 0 for the zero-mean
# input would collapse every scale to the floor and saturate the output.
# Mirrors tests/kernels/helion/test_silu_and_mul_per_block_quant.py.
act_abs = SiluAndMul.forward_native(input.to(torch.float32)).abs()
scale_ub = (0.5 * (act_abs.mean() + act_abs.amax())).to(scale_dtype)
config_key = CaseKey(
{
"intermediate_size": intermediate_size,
"group_size": group_size,
"num_tokens": num_tokens,
}
)
inputs[config_key] = (result, input, scale, group_size, scale_ub, False)
return inputs
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest intermediate_size among available configs
(exact match preferred).
2. Find the closest group_size among available configs
(exact match preferred).
3. Among the num_tokens values tuned for that intermediate_size and group_size,
pick the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
result, _, _, group_size, *_ = args
num_tokens, intermediate_size = result.shape
cache_key = (num_tokens, group_size, intermediate_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, dict[int, list[int]]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["intermediate_size"], {}).setdefault(
key["group_size"], []
).append(key["num_tokens"])
if not configs:
return None
best_intermediate_size = min(configs, key=lambda s: abs(s - intermediate_size))
best_group_size = min(
configs[best_intermediate_size], key=lambda s: abs(s - group_size)
)
available_num_tokens = sorted(configs[best_intermediate_size][best_group_size])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey(
{
"intermediate_size": best_intermediate_size,
"group_size": best_group_size,
"num_tokens": best_num_tokens,
}
)
_pick_cache[cache_key] = result
return result
def baseline(
out: torch.Tensor, # [num_tokens, intermediate_size]
input: torch.Tensor, # [num_tokens, 2 * intermediate_size]
scales: torch.Tensor, # [num_tokens, groups_per_row]
group_size: int,
scale_ub: torch.Tensor | None = None, # scalar tensor
is_scale_transposed: bool = False,
) -> None:
num_tokens, intermediate_size = out.shape
groups_per_row = intermediate_size // group_size
quant_dtype = out.dtype
qtype_min: int | float
qtype_max: int | float
if quant_dtype == torch.int8:
qtype_min, qtype_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_min, qtype_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_max * 512.0)
act = SiluAndMul.forward_native(input.to(torch.float32))
x_grouped = act.view(num_tokens, groups_per_row, group_size)
s = torch.amax(torch.abs(x_grouped), dim=-1)
if scale_ub is not None:
s = s.clamp(max=scale_ub)
s = (s * (1.0 / qtype_max)).clamp(min=min_scaling_factor)
y = x_grouped / s[:, :, None]
if quant_dtype == torch.int8:
y = y.round()
scales.copy_(s)
out.copy_(
y.clamp(qtype_min, qtype_max).view(num_tokens, intermediate_size).to(out.dtype)
)
@register_kernel(
mutates_args=["out", "scales"],
config_picker=pick_config,
input_generator=generate_inputs,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
ignore_warnings=[helion.exc.TensorOperationInWrapper],
),
) # type: ignore[misc]
def silu_and_mul_per_block_quant(
out: torch.Tensor, # [num_tokens, intermediate_size]
input: torch.Tensor, # [num_tokens, 2 * intermediate_size]
scales: torch.Tensor, # [num_tokens, groups_per_row]
group_size: int,
scale_ub: torch.Tensor | None = None, # scalar tensor
is_scale_transposed: bool = False, # dummy
) -> None:
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
num_tokens, two_intermediate_size = input.shape
hl.specialize(two_intermediate_size)
assert two_intermediate_size % 2 == 0
intermediate_size = two_intermediate_size // 2
assert out.shape[0] == num_tokens
assert out.shape[1] == intermediate_size
fp8_dtype = get_fp8_dtype()
assert out.dtype in [fp8_dtype, torch.int8]
if scale_ub is not None:
assert out.dtype == fp8_dtype
assert scale_ub.dtype == torch.float32
assert scales.ndim == 2 and scales.dtype == torch.float32
assert scales.shape[0] == num_tokens
groups_per_row = scales.shape[1]
hl.specialize(groups_per_row)
assert (
intermediate_size % group_size == 0
and intermediate_size // group_size == groups_per_row
)
assert group_size in [64, 128]
hl.specialize(group_size)
assert input.stride()[-1] == 1
assert out.stride()[-1] == 1
quant_dtype = out.dtype
qtype_traits_min: int | float
qtype_traits_max: int | float
if quant_dtype == torch.int8:
qtype_traits_min, qtype_traits_max = get_int8_min_max()
min_scaling_factor = get_int8_min_scaling_factor()
else:
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
min_scaling_factor = 1.0 / (qtype_traits_max * 512.0)
qtype_max = float(qtype_traits_max)
input = input.view(num_tokens, -1, group_size)
out = out.view(num_tokens, -1, group_size)
for tile_m, tile_gn, tile_n in hl.tile(
[num_tokens, groups_per_row, group_size], block_size=[1, None, group_size]
):
x_a_blk = input[tile_m, tile_gn, tile_n].to(torch.float32)
x_b_blk = hl.load(
input,
[tile_m, tile_gn.index + groups_per_row, tile_n],
extra_mask=(tile_gn.index + groups_per_row < 2 * groups_per_row)[
None, :, None
],
).to(torch.float32)
x_blk = x_a_blk * torch.sigmoid(x_a_blk) * x_b_blk
s_blk = torch.amax(torch.abs(x_blk), dim=-1).to(torch.float32)
if scale_ub is not None:
scale_ub_s = hl.load(scale_ub, [])
s_blk = s_blk.clamp(max=scale_ub_s)
s_blk = s_blk * (1.0 / qtype_max)
s_blk = s_blk.clamp(min=min_scaling_factor)
scales[tile_m, tile_gn] = s_blk
if quant_dtype == torch.int8:
y_blk = (x_blk * (1.0 / s_blk[:, :, None])).round()
else:
y_blk = x_blk / s_blk[:, :, None]
out[tile_m, tile_gn, tile_n] = y_blk.clamp(
qtype_traits_min, qtype_traits_max
).to(out.dtype)
-135
View File
@@ -1,135 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"silu_mul_fp8 Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion.language as hl
from vllm.kernels.helion.register import register_kernel
logger = init_logger(__name__)
def generate_silu_mul_fp8_inputs() -> dict[CaseKey, tuple[Any, ...]]:
intermediate_sizes = [2048, 2880, 4096, 8192, 11008, 14336]
# Use the same num_tokens values as vLLM's default cudagraph capture sizes.
# See vllm/config/vllm.py _set_cudagraph_sizes() for the canonical formula.
num_tokens_list = [1, 2, 4] + list(range(8, 256, 8)) + list(range(256, 513, 16))
inputs: dict[CaseKey, tuple[Any, ...]] = {}
for num_tokens in num_tokens_list:
for intermediate_size in intermediate_sizes:
input_tensor = torch.randn(
num_tokens,
2 * intermediate_size,
device="cuda",
dtype=torch.bfloat16,
)
scale = torch.tensor([1.0], device="cuda", dtype=torch.float32)
key = CaseKey({"intermediate": intermediate_size, "numtokens": num_tokens})
inputs[key] = (input_tensor, scale)
return inputs
_pick_cache: dict[tuple[int, int], CaseKey | None] = {}
def pick_silu_mul_fp8_config(
args: tuple[Any, ...], config_keys: list[CaseKey]
) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest intermediate_size among available configs
(exact match preferred).
2. Among the num_tokens values tuned for that intermediate_size, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
input_tensor, _scale = args
intermediate_size = int(input_tensor.shape[-1]) // 2
num_tokens = int(input_tensor.view(-1, input_tensor.shape[-1]).shape[0])
cache_key = (num_tokens, intermediate_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
by_isize: dict[int, list[int]] = {}
for k in config_keys:
if k.is_default():
continue
by_isize.setdefault(k["intermediate"], []).append(k["numtokens"])
if not by_isize:
return None
best_isize = min(by_isize, key=lambda s: abs(s - intermediate_size))
available = sorted(by_isize[best_isize])
best_ntokens = next((n for n in available if n >= num_tokens), available[-1])
result = CaseKey({"intermediate": best_isize, "numtokens": best_ntokens})
_pick_cache[cache_key] = result
return result
@register_kernel(
config_picker=pick_silu_mul_fp8_config,
input_generator=generate_silu_mul_fp8_inputs,
)
def silu_mul_fp8(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
original_shape = input.shape
two_d = hl.specialize(original_shape[-1])
d = two_d // 2
output_shape = original_shape[:-1] + (d,)
input_2d = input.view(-1, original_shape[-1])
m = input_2d.shape[0]
# TODO(gmagogsfm): Support for more float8 subtypes (e4m3fnuz, e5m2) coming
out = torch.empty((m, d), device=input.device, dtype=torch.float8_e4m3fn)
input_part_a = input_2d[:, :d]
input_part_b = input_2d[:, d:]
assert scale.numel() == 1, "Scale must be a scalar Tensor"
for tile_m, tile_n in hl.tile([m, d]):
a_vals = input_part_a[tile_m, tile_n]
silu_result = torch.nn.functional.silu(a_vals)
b_vals = input_part_b[tile_m, tile_n]
result = silu_result * b_vals
result_f32 = result.to(torch.float32)
scale_val = hl.load(scale, [0])
inv_scale = 1.0 / scale_val
result_scaled = result_f32 * inv_scale
out[tile_m, tile_n] = result_scaled.to(out.dtype)
return out.view(output_shape)
def silu_mul_fp8_baseline(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
output_shape = input.shape[:-1] + (input.shape[-1] // 2,)
out = torch.empty(output_shape, dtype=torch.float8_e4m3fn, device=input.device)
torch.ops._C.silu_and_mul_quant(out, input, scale)
return out
-513
View File
@@ -1,513 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM Helion kernel registration with pre-tuned config selection.
This module leverages Helion's internal config selection infrastructure to use
pre-tuned configs instead of runtime autotuning.
How Helion Normally Works
-------------------------
For each kernel invocation, Helion:
1. Computes a cache key from input arguments
2. Looks up the key in its internal compilation cache
3. On cache miss, runs autotuning to find the best config
4. Compiles and caches the kernel with that config
How We Override It
------------------
We override two Helion hooks to use pre-tuned configs:
1. **key**: We provide a key function (derived from config_picker) that
computes cache keys matching our pre-tuned config keys. This ensures Helion's
internal cache uses keys that correspond to configs we've prepared.
2. **autotuner_fn**: We provide PresetConfigSearch which, instead of autotuning,
simply returns the pre-tuned config for the computed key. On cache miss,
Helion calls our autotuner which returns the author-prepared config.
Both hooks use the same config_picker logic to ensure the cache key computed
by key matches the config returned by the autotuner.
Key Classes
-----------
- HelionKernelWrapper: Wraps raw kernel + config_picker, creates configured kernels
- ConfiguredHelionKernel: Platform-specific kernel with pre-tuned configs
- PresetConfigSearch: Custom autotuner that returns pre-tuned configs
"""
from __future__ import annotations
import inspect
from collections.abc import Callable
from typing import Any
import torch
from torch.library import Library
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
from vllm.utils.torch_utils import direct_register_custom_op
if not has_helion():
raise ImportError(
"register module requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
from helion.autotuner.base_search import BaseAutotuner
from helion.runtime.config import Config
from helion.runtime.settings import default_autotuner_fn
# TODO(gmagogsfm): Remove CustomOp fallback path (_get_or_register_custom_op,
# vllm_helion_lib, direct_register_custom_op) once vLLM requires PyTorch >= 2.11.
# FIXME(gmagogsfm): Re-enable HOP path once performance regression is fixed.
# _HOP_AVAILABLE = requires_torch_version("2.11")
_HOP_AVAILABLE = False
if _HOP_AVAILABLE:
from helion._compat import supports_torch_compile_fusion
from helion._compiler._dynamo.higher_order_ops import helion_kernel_side_table
from helion._compiler._dynamo.variables import HelionKernelVariable
from helion.runtime.kernel import Kernel
from torch._dynamo.guards import GuardBuilder
from torch._dynamo.variables.builder import VariableBuilder
logger = init_logger(__name__)
vllm_helion_lib = Library("vllm_helion", "FRAGMENT") # noqa
ConfigPicker = Callable[[tuple[Any, ...], list[CaseKey]], CaseKey | None]
def validate_helion_settings(
helion_settings: helion.Settings | None, op_name: str
) -> None:
if helion_settings is None:
return
settings_dict = helion_settings.to_dict()
if (
"autotuner_fn" in settings_dict
and settings_dict["autotuner_fn"] is not None
and settings_dict["autotuner_fn"] is not default_autotuner_fn
):
raise ValueError(
f"HelionKernelWrapper for '{op_name}' uses a custom autotuner via "
f"config picker. Remove 'autotuner_fn' from helion_settings and use "
f"register_kernel(..., config_picker=...) instead."
)
if settings_dict.get("static_shapes") is True:
logger.warning(
"Kernel '%s' has static_shapes=True in helion_settings, "
"which will be overridden to False. vLLM requires dynamic "
"shapes for variable batch sizes and sequence lengths.",
op_name,
)
def create_helion_decorated_kernel(
raw_kernel_func: Callable,
helion_settings: helion.Settings | None = None,
extra_kwargs: dict[str, Any] | None = None,
) -> Any:
kernel_kwargs: dict[str, Any] = {}
if helion_settings:
kernel_kwargs.update(helion_settings.to_dict())
# vLLM requires dynamic shapes for variable batch sizes and sequence lengths
kernel_kwargs["static_shapes"] = False
if extra_kwargs:
kernel_kwargs.update(extra_kwargs)
return helion.kernel(**kernel_kwargs)(raw_kernel_func)
class PresetConfigSearch(BaseAutotuner):
"""Custom autotuner that uses a preset config selector instead of autotuning."""
def __init__(
self,
args: tuple[Any, ...],
config_selector: Callable[[tuple[Any, ...]], Config],
):
self.args = args
self.config_selector = config_selector
def autotune(self, *, skip_cache: bool = False) -> Config:
return self.config_selector(self.args)
class ConfiguredHelionKernel:
"""A configured Helion kernel bound to a specific platform."""
def __init__(
self,
op_name: str,
config_picker: ConfigPicker | None,
raw_kernel_func: Callable,
helion_settings: helion.Settings | None = None,
):
self.op_name = op_name
self.config_picker = config_picker
self.raw_kernel_func = raw_kernel_func
self.helion_settings = helion_settings
self._decorated_kernel = self._create_decorated_kernel()
def __call__(self, *args, **kwargs):
return self._decorated_kernel(*args, **kwargs)
def _create_key_computer(self):
"""Create a key computer function derived from the config picker.
The returned function receives kernel arguments unpacked (*args) to match
Helion's key signature (called as self._key_fn(*args)).
"""
if self.config_picker is None:
raise RuntimeError(
f"No config picker registered for kernel '{self.op_name}'. "
f"A config_picker must be provided to register_kernel()."
)
picker = self.config_picker
all_keys = list(self.configs.keys())
default = CaseKey.default()
has_default = default in self.configs
def key_computer(*args):
selected = picker(args, all_keys)
if selected is not None:
return str(selected)
if has_default:
return str(default)
return None
return key_computer
def _create_config_selector(self, key_computer):
str_to_key = {str(k): k for k in self.configs}
def config_selector(args):
selected_str = key_computer(*args)
if selected_str is None:
raise ValueError(
f"Config picker returned None for kernel "
f"'{self.op_name}' with available config keys: "
f"{list(self.configs.keys())}"
)
config_key = str_to_key.get(selected_str)
if config_key is None:
raise ValueError(
f"Config picker returned invalid config key "
f"'{selected_str}' for kernel "
f"'{self.op_name}'. "
f"Available keys: {list(self.configs.keys())}"
)
return self.configs[config_key]
return config_selector
def _load_platform_configs(self) -> None:
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.utils import get_canonical_gpu_name
self.platform = get_canonical_gpu_name()
config_manager = ConfigManager()
self.configs = config_manager.get_platform_configs(self.op_name, self.platform)
if not self.configs:
raise ValueError(
f"No configs available for kernel '{self.op_name}' "
f"on platform '{self.platform}'"
)
def _create_decorated_kernel(self) -> Callable[..., Any]:
self._load_platform_configs()
key_computer = self._create_key_computer()
config_selector = self._create_config_selector(key_computer)
extra_kwargs = {
"autotuner_fn": lambda _, args: PresetConfigSearch(args, config_selector),
"key": key_computer,
}
logger.debug(
"Creating decorated kernel %s with custom autotuner on platform %s",
self.op_name,
self.platform,
)
return create_helion_decorated_kernel(
self.raw_kernel_func, self.helion_settings, extra_kwargs
)
class HelionKernelWrapper:
"""Wrapper for Helion kernels with pre-tuned config selection and HOP support."""
def __init__(
self,
raw_kernel_func: Callable,
op_name: str,
fake_impl: Callable | None,
config_picker: ConfigPicker,
mutates_args: list[str] | None = None,
helion_settings: helion.Settings | None = None,
input_generator: (Callable[[], dict[CaseKey, tuple[Any, ...]]] | None) = None,
):
# Validate helion_settings doesn't conflict with our custom autotuner
validate_helion_settings(helion_settings, op_name)
self.raw_kernel_func = raw_kernel_func
self.op_name = op_name
self._fake_impl = fake_impl
self.helion_settings = helion_settings
self._config_picker = config_picker
self._input_generator = input_generator
self._mutates_args = mutates_args
self._configured_kernel: ConfiguredHelionKernel | None = None
# TODO(@gmagogsfm): Remove this disable flag once integrated with vLLM IR,
# which handles op enablement/disablement.
self._disabled = False
self._disabled_reason: str | None = None
try:
if not _HOP_AVAILABLE:
self._get_or_register_custom_op()
else:
self.get_configured_op()
except ValueError as e:
self._disabled = True
self._disabled_reason = str(e)
logger.warning(
"Helion kernel '%s' is disabled: %s",
op_name,
self._disabled_reason,
)
def __call__(self, *args, **kwargs):
if self._disabled:
raise RuntimeError(
f"Helion kernel '{self.op_name}' is disabled: {self._disabled_reason}"
)
if not _HOP_AVAILABLE:
op = getattr(torch.ops.vllm_helion, self.op_name)
return op(*args, **kwargs)
assert self._configured_kernel is not None, (
f"Kernel '{self.op_name}' was not initialized. "
"Please open an issue on GitHub."
)
# During Dynamo tracing, this call will be intercepted by our custom
# HelionKernelWrapperVariable and handled via proper HOP emission.
# During eager execution, call the kernel directly.
return self._configured_kernel(*args, **kwargs)
def get_inputs(self) -> dict[CaseKey, tuple[Any, ...]]:
if self._input_generator is None:
raise NotImplementedError(
f"No input generator registered for kernel '{self.op_name}'. "
f"Use register_kernel(..., input_generator=...) to register one."
)
return self._input_generator()
def run_autotune(
self,
inputs: tuple[Any, ...],
autotune_effort: str = "quick",
) -> Config:
"""Run autotuning for a single input configuration."""
extra_kwargs = {
"autotune_effort": autotune_effort,
"autotune_ignore_errors": True,
}
autotune_kernel = create_helion_decorated_kernel(
self.raw_kernel_func, self.helion_settings, extra_kwargs
)
return autotune_kernel.autotune(inputs)
def get_configured_op(self) -> ConfiguredHelionKernel:
if self._disabled:
raise RuntimeError(
f"Helion kernel '{self.op_name}' is disabled: {self._disabled_reason}"
)
if self._configured_kernel is None:
self._configured_kernel = ConfiguredHelionKernel(
op_name=self.op_name,
config_picker=self._config_picker,
raw_kernel_func=self.raw_kernel_func,
helion_settings=self.helion_settings,
)
return self._configured_kernel
def _get_or_register_custom_op(self) -> Any:
if hasattr(torch.ops.vllm_helion, self.op_name):
return getattr(torch.ops.vllm_helion, self.op_name)
configured_kernel = self.get_configured_op()
logger.info("Registering op: vllm_helion::%s", self.op_name)
direct_register_custom_op(
op_name=self.op_name,
op_func=configured_kernel._decorated_kernel,
mutates_args=self._mutates_args,
fake_impl=self._fake_impl,
target_lib=vllm_helion_lib,
)
return getattr(torch.ops.vllm_helion, self.op_name)
# Global registry for tracking all registered HelionKernelWrapper instances
_REGISTERED_KERNELS: dict[str, HelionKernelWrapper] = {}
def get_registered_kernels() -> dict[str, HelionKernelWrapper]:
return _REGISTERED_KERNELS.copy()
def get_kernel_by_name(kernel_name: str) -> HelionKernelWrapper | None:
return _REGISTERED_KERNELS.get(kernel_name)
def infer_fake_impl(
kernel_func: Callable,
helion_settings: helion.Settings | None = None,
) -> Callable:
def helion_fake_kernel(*args, **kwargs):
kernel_kwargs = {}
if helion_settings:
kernel_kwargs.update(helion_settings.to_dict())
temp_decorated_kernel = helion.kernel(**kernel_kwargs)(kernel_func)
# Bind with args to get config_spec, then get a valid default config
bound = temp_decorated_kernel.bind(args)
default_config = bound.config_spec.default_config()
compiled_runner = bound.compile_config(default_config)
return compiled_runner(*args, **kwargs, _launcher=lambda *a, **kw: None)
return helion_fake_kernel
def register_kernel(
op_name: str | None = None,
*,
config_picker: ConfigPicker,
fake_impl: Callable | None = None,
mutates_args: list[str] | None = None,
helion_settings: helion.Settings | None = None,
input_generator: (Callable[[], dict[CaseKey, tuple[Any, ...]]] | None) = None,
) -> Callable[[Callable], HelionKernelWrapper]:
"""Register a Helion kernel with pre-tuned config selection.
Args:
op_name: Name to register the op under. Defaults to the function name.
fake_impl: Optional meta/fake implementation for tracing.
mutates_args: Names of arguments the kernel mutates in place.
helion_settings: Optional Helion settings override.
config_picker: Required. Receives ``(args, config_keys)``
where each config key is a ``dict[str, Any]`` mapping
parameter names to values. Return the best-matching
dict, or ``None`` to fall back to the default config.
Example::
def pick_config(args, config_keys):
x = args[0]
best = min(config_keys, key=lambda k: abs(k["size"] - x.shape[0]))
return best
input_generator: Optional. Returns ``dict[str, tuple]`` where
each key is a serialized config key and each value is a
tuple of arguments to pass to the kernel.
Example::
def generate_inputs():
return {
"4096": (torch.randn(4096, device="cuda"), 0.5),
"8192": (torch.randn(8192, device="cuda"), 0.5),
}
"""
def decorator(kernel_func: Callable) -> HelionKernelWrapper:
final_op_name = op_name if op_name else kernel_func.__name__
if final_op_name in _REGISTERED_KERNELS:
raise ValueError(
f"Helion kernel '{final_op_name}' is already registered. "
f"Use a different op_name or check for duplicate registrations."
)
# A void-returning kernel that mutates its args needs no fake impl:
# PyTorch auto-generates a trivial one for mutable, no-return schemas.
returns_none = inspect.signature(kernel_func).return_annotation is None
final_fake_impl = fake_impl
if final_fake_impl is None and not (mutates_args and returns_none):
final_fake_impl = infer_fake_impl(kernel_func, helion_settings)
logger.debug(
"Auto-generated fake_impl for Helion kernel '%s'",
kernel_func.__name__,
)
kernel_wrapper = HelionKernelWrapper(
raw_kernel_func=kernel_func,
op_name=final_op_name,
fake_impl=final_fake_impl,
config_picker=config_picker,
mutates_args=mutates_args,
helion_settings=helion_settings,
input_generator=input_generator,
)
_REGISTERED_KERNELS[final_op_name] = kernel_wrapper
logger.info(
"Registered Helion kernel '%s' as HelionKernelWrapper",
kernel_func.__name__,
)
return kernel_wrapper
return decorator
# Register HelionKernelWrapper with Dynamo's variable tracker system
if _HOP_AVAILABLE:
def _register_vllm_helion_dynamo_variable():
"""Register HelionKernelWrapper with Dynamo's VariableBuilder.
When Dynamo encounters a HelionKernelWrapper during tracing, this
extracts the underlying Helion Kernel and delegates to Helion's own
registered Kernel handler, which handles HOP emission, side table
registration, and inductor lowering setup.
"""
def wrap_helion_kernel_wrapper(
builder: VariableBuilder, value: HelionKernelWrapper
):
kernel = value.get_configured_op()._decorated_kernel
if supports_torch_compile_fusion():
helion_handler = VariableBuilder._type_dispatch()[Kernel]
return helion_handler(builder, kernel)
kernel_idx = helion_kernel_side_table.add_kernel(kernel)
builder.install_guards(GuardBuilder.ID_MATCH)
return HelionKernelVariable(kernel, kernel_idx, source=builder.source)
dispatch = VariableBuilder._type_dispatch()
dispatch[HelionKernelWrapper] = wrap_helion_kernel_wrapper
# Register immediately when the module is imported
_register_vllm_helion_dynamo_variable()
-93
View File
@@ -1,93 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility functions for Helion kernel management."""
import regex as re
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
logger = init_logger(__name__)
# Maps known variant GPU names (after lowercase/underscore normalization)
# to their canonical form.
#
# Names that are already canonical after normalization are NOT listed here.
# For example, "NVIDIA H200" normalizes to "nvidia_h200" which needs no
# further mapping, and AMD ROCm names like "AMD_Instinct_MI300X" come from
# a controlled lookup table in rocm.py and normalize cleanly to
# "amd_instinct_mi300x". Only names with variant suffixes (form factor,
# memory size, memory type, etc.) that should be stripped need entries.
#
# To add a new GPU variant: run `canonicalize_gpu_name()` without the alias
# to see the normalized name, then add a mapping here if it contains variant
# suffixes that should be stripped (e.g. Blackwell/Rubin variants).
_GPU_NAME_ALIASES: dict[str, str] = {
# H100 variants
"nvidia_h100_pcie": "nvidia_h100",
"nvidia_h100_sxm5": "nvidia_h100",
"nvidia_h100_80gb_hbm3": "nvidia_h100",
"nvidia_h100_nvl": "nvidia_h100",
# H200 variants
"nvidia_h200_nvl": "nvidia_h200",
"nvidia_h200_141gb_hbm3e": "nvidia_h200",
# A100 variants
"nvidia_a100_sxm4_80gb": "nvidia_a100",
"nvidia_a100_sxm4_40gb": "nvidia_a100",
"nvidia_a100_pcie_80gb": "nvidia_a100",
"nvidia_a100_pcie_40gb": "nvidia_a100",
"nvidia_a100_80gb_pcie": "nvidia_a100",
# V100 variants (Tesla-branded)
"tesla_v100_sxm2_32gb": "tesla_v100",
"tesla_v100_sxm2_16gb": "tesla_v100",
"tesla_v100_pcie_32gb": "tesla_v100",
"tesla_v100_pcie_16gb": "tesla_v100",
# AMD ROCm variants (from _ROCM_DEVICE_ID_NAME_MAP in rocm.py)
"amd_instinct_mi300x_hf": "amd_instinct_mi300x",
# ADD MORE HERE
}
def get_gpu_name(device_id: int | None = None) -> str:
if device_id is None:
logger.warning_once(
"get_gpu_name() called without device_id, defaulting to 0. "
"This may return the wrong device name in multi-node setups."
)
device_id = 0
return current_platform.get_device_name(device_id)
def canonicalize_gpu_name(name: str) -> str:
"""Canonicalize GPU name for use as a platform identifier.
Converts to lowercase, replaces separators with underscores,
and maps known variant names to their canonical form via _GPU_NAME_ALIASES.
e.g., "NVIDIA H100 80GB HBM3" -> "nvidia_h100"
"NVIDIA A100-SXM4-80GB" -> "nvidia_a100"
"AMD Instinct MI300X" -> "amd_instinct_mi300x"
"""
if not name or not name.strip():
raise ValueError("GPU name cannot be empty")
name = re.sub(r"[\s/-]+", "_", name.lower())
if name in _GPU_NAME_ALIASES:
return _GPU_NAME_ALIASES[name]
return name
def get_canonical_gpu_name(device_id: int | None = None) -> str:
return canonicalize_gpu_name(get_gpu_name(device_id))
def get_fp8_dtype() -> torch.dtype:
return current_platform.fp8_dtype()
def get_int8_min_max() -> tuple[int, int]:
qtype_traits = torch.iinfo(torch.int8)
return qtype_traits.min, qtype_traits.max
def get_int8_min_scaling_factor() -> float:
return torch.finfo(torch.float32).eps
-127
View File
@@ -1,127 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""This file registers Oink implementations for vLLM IR ops.
vLLM does not depend on the external Oink repository/package. When an external
plugin registers torch.library.custom_op entrypoints under the `oink::`
namespace (e.g. via vLLM's general_plugins mechanism), these ops will be marked
as supported. To dispatch to those ops, set kernel_config.ir_op_priority.<op> to oink.
Alternatively, `VLLM_USE_OINK_OPS=1` will add this to priority by default.
"""
import torch
from torch import Tensor
from vllm import ir
from vllm.platforms import current_platform
OINK_AVAILABLE = current_platform.has_device_capability(100) and hasattr(
torch.ops, "oink"
)
def has_oink_op(name: str) -> bool:
"""Check if a specific oink op is registered."""
return OINK_AVAILABLE and hasattr(torch.ops.oink, name)
def _can_view_as_2d(x: Tensor) -> bool:
"""Return True if x.view(-1, x.shape[-1]) is viewable (no copy)."""
if x.dim() < 2:
return False
if x.dim() == 2:
return True
# For a view(-1, N) to be valid, all leading dims must be contiguous with
# respect to each other (size-1 dims are ignored).
for dim in range(x.dim() - 1):
# Strides for size-1 dims are irrelevant and can be arbitrary.
if x.size(dim + 1) != 1 and x.stride(dim) != x.stride(dim + 1) * x.size(
dim + 1
):
return False
return True
def _is_oink_stride_compatible_2d(x_2d: Tensor) -> bool:
"""Return True if x_2d meets Oink's pointer-path stride constraints."""
if x_2d.dim() != 2:
return False
if x_2d.stride(1) != 1:
return False
# Match Oink's vectorization constraint: stride(0) divisible by 256b.
if x_2d.dtype in (torch.float16, torch.bfloat16):
divby = 16
elif x_2d.dtype == torch.float32:
divby = 8
else:
return False
return (x_2d.stride(0) % divby) == 0
oink_rms_supported = (
lambda x, weight, epsilon, variance_size=None: variance_size is None
and weight is not None
and x.dim() >= 2
and x.dtype == weight.dtype
and weight.is_contiguous()
and _can_view_as_2d(x)
and _is_oink_stride_compatible_2d(x.view(-1, x.shape[-1]))
)
"""
Oink rms only supports 2d-like inputs with contiguous weight
and no variance_size override.
"""
@ir.ops.rms_norm.register_impl(
"oink", supports_args=oink_rms_supported, supported=has_oink_op("rmsnorm")
)
def rms_norm(
x: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> Tensor:
assert variance_size is None
x_2d = x.view(-1, x.shape[-1])
return torch.ops.oink.rmsnorm(x_2d, weight, epsilon).view_as(x)
oink_add_rms_supported = (
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
and weight is not None
and x.dim() >= 2
and x.dtype == weight.dtype
and weight.is_contiguous()
and _can_view_as_2d(x)
and _is_oink_stride_compatible_2d(x.view(-1, x.shape[-1]))
# residual must have 2d-compatible strides and match x shape/dtype
and x.dtype == x_residual.dtype
and x.shape == x_residual.shape
and _can_view_as_2d(x_residual)
and _is_oink_stride_compatible_2d(x_residual.view(-1, x_residual.shape[-1]))
)
"""
Oink fused_add_rms_norm has the same constraints as rms_norm,
and residual must be 2d-like with compatible strides.
"""
@ir.ops.fused_add_rms_norm.register_impl(
"oink",
supports_args=oink_add_rms_supported,
supported=has_oink_op("fused_add_rms_norm"),
inplace=True,
)
def fused_add_rms_norm(
x: Tensor,
x_residual: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> tuple[Tensor, Tensor]:
assert variance_size is None
x_2d = x.view(-1, x.shape[-1])
residual_2d = x_residual.view(-1, x_residual.shape[-1])
torch.ops.oink.fused_add_rms_norm(x_2d, residual_2d, weight, epsilon)
return x, x_residual