[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -2,6 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kernel implementations for vLLM."""
|
||||
|
||||
from . import aiter_ops, oink_ops, triton, vllm_c
|
||||
from . import aiter_ops, triton, vllm_c
|
||||
|
||||
__all__ = ["vllm_c", "aiter_ops", "oink_ops", "triton"]
|
||||
__all__ = ["vllm_c", "aiter_ops", "triton"]
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Helion integration for vLLM."""
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import (
|
||||
ConfigManager,
|
||||
ConfigSet,
|
||||
)
|
||||
from vllm.kernels.helion.register import (
|
||||
ConfigPicker,
|
||||
ConfiguredHelionKernel,
|
||||
HelionKernelWrapper,
|
||||
get_kernel_by_name,
|
||||
get_registered_kernels,
|
||||
register_kernel,
|
||||
vllm_helion_lib,
|
||||
)
|
||||
from vllm.kernels.helion.utils import canonicalize_gpu_name, get_canonical_gpu_name
|
||||
|
||||
__all__ = [
|
||||
# Config management
|
||||
"CaseKey",
|
||||
"ConfigManager",
|
||||
"ConfigSet",
|
||||
# Kernel registration
|
||||
"ConfigPicker",
|
||||
"ConfiguredHelionKernel",
|
||||
"HelionKernelWrapper",
|
||||
"get_kernel_by_name",
|
||||
"get_registered_kernels",
|
||||
"register_kernel",
|
||||
"vllm_helion_lib",
|
||||
# Utilities
|
||||
"canonicalize_gpu_name",
|
||||
"get_canonical_gpu_name",
|
||||
]
|
||||
@@ -1,70 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Structured key for identifying kernel config/autotune/benchmark cases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
|
||||
class CaseKey(dict[str, Any]):
|
||||
"""Immutable, hashable dict for identifying kernel cases.
|
||||
|
||||
Used as the key for config lookup, autotuning, benchmarking, and
|
||||
input generation. Behaves like a read-only dict and can be used
|
||||
as a dict key or in sets.
|
||||
|
||||
The canonical string form (``__str__``) is stable JSON with sorted
|
||||
keys. Use ``CaseKey.default()`` for the default/fallback key.
|
||||
The regular constructor requires at least one key-value pair::
|
||||
|
||||
CaseKey({"intermediate": 2048, "numtokens": 256})
|
||||
CaseKey.default() # default/fallback
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, _allow_empty: bool = False, **kwargs: Any):
|
||||
super().__init__(*args, **kwargs)
|
||||
if not self and not _allow_empty:
|
||||
raise TypeError(
|
||||
"CaseKey requires at least one key-value pair. "
|
||||
"Use CaseKey.default() for the default config key."
|
||||
)
|
||||
self._str: str | None = None
|
||||
self._hash: int | None = None
|
||||
|
||||
@classmethod
|
||||
def default(cls) -> CaseKey:
|
||||
"""Create a default case key (empty)."""
|
||||
return cls(_allow_empty=True)
|
||||
|
||||
def __hash__(self) -> int: # type: ignore[override]
|
||||
if self._hash is None:
|
||||
self._hash = hash(str(self))
|
||||
return self._hash
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self._str is None:
|
||||
self._str = json.dumps(dict(self), sort_keys=True, separators=(",", ":"))
|
||||
return self._str
|
||||
|
||||
def __repr__(self) -> str:
|
||||
if not self:
|
||||
return "CaseKey.default()"
|
||||
return f"CaseKey({dict(self)})"
|
||||
|
||||
def is_default(self) -> bool:
|
||||
"""Return True if this is the default case key (empty)."""
|
||||
return not self
|
||||
|
||||
def _readonly(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise TypeError("CaseKey is immutable")
|
||||
|
||||
__setitem__ = _readonly # type: ignore[assignment]
|
||||
__delitem__ = _readonly # type: ignore[assignment]
|
||||
__ior__ = _readonly # type: ignore[assignment]
|
||||
update = _readonly # type: ignore[assignment]
|
||||
pop = _readonly # type: ignore[assignment]
|
||||
popitem = _readonly # type: ignore[assignment]
|
||||
setdefault = _readonly # type: ignore[assignment]
|
||||
clear = _readonly # type: ignore[assignment]
|
||||
@@ -1,321 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Configuration management for Helion kernels.
|
||||
|
||||
This module provides centralized configuration file management for Helion custom
|
||||
operations, including naming conventions, directory resolution, and file I/O.
|
||||
|
||||
Config File Structure
|
||||
---------------------
|
||||
Each kernel has a directory: {kernel_name}/
|
||||
Inside, each GPU platform has its own JSON file: {kernel_name}/{platform}.json
|
||||
|
||||
Platform files store config entries as a JSON array::
|
||||
|
||||
[
|
||||
{"key": {}, "config": {...}},
|
||||
{"key": {"intermediate": 2048, "numtokens": 256}, "config": {...}},
|
||||
...,
|
||||
]
|
||||
|
||||
Config keys are ``CaseKey`` instances mapping parameter names to
|
||||
values. The default config uses ``CaseKey.default()``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
raise ImportError(
|
||||
"ConfigManager requires helion to be installed. "
|
||||
"Install it with: pip install helion"
|
||||
)
|
||||
|
||||
import helion
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ConfigSet:
|
||||
"""In-memory collection of Helion configs with lookup/query capabilities.
|
||||
|
||||
Configs are stored keyed by ``CaseKey``. The default config
|
||||
uses ``CaseKey.default()`` as its key.
|
||||
"""
|
||||
|
||||
_ConfigDict = dict[str, dict[CaseKey, "helion.Config"]]
|
||||
|
||||
def __init__(self, kernel_name: str):
|
||||
self._kernel_name = kernel_name
|
||||
self._configs: ConfigSet._ConfigDict = {}
|
||||
|
||||
@property
|
||||
def kernel_name(self) -> str:
|
||||
return self._kernel_name
|
||||
|
||||
def get_config(self, platform: str, config_key: CaseKey) -> helion.Config:
|
||||
platform_dict = self._configs.get(platform)
|
||||
if platform_dict is None:
|
||||
avail_platforms = self.get_platforms()
|
||||
# TODO(@gmagogsfm): add a CLI/env override flag so users can
|
||||
# directly specify a platform name instead of relying on
|
||||
# auto-detection, and suggest it in this error message.
|
||||
raise KeyError(
|
||||
f"Config not found for kernel '{self._kernel_name}': "
|
||||
f"platform '{platform}' not found. "
|
||||
f"Available platforms: {avail_platforms or '(none)'}. "
|
||||
f"If your GPU is a variant of a supported platform, "
|
||||
f"consider adding a mapping in _GPU_NAME_ALIASES in "
|
||||
f"vllm/kernels/helion/utils.py, or run "
|
||||
f"tools/autotune_helion_kernels.py to generate configs "
|
||||
f"for your platform."
|
||||
)
|
||||
|
||||
config = platform_dict.get(config_key)
|
||||
if config is None:
|
||||
avail_keys = self.get_config_keys(platform)
|
||||
raise KeyError(
|
||||
f"Config not found for kernel '{self._kernel_name}': "
|
||||
f"config_key '{config_key}' not found for "
|
||||
f"platform '{platform}'. "
|
||||
f"Available config_keys: {avail_keys or '(none)'}"
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
def get_platforms(self) -> list[str]:
|
||||
return sorted(self._configs.keys())
|
||||
|
||||
def get_config_keys(self, platform: str) -> list[CaseKey]:
|
||||
platform_dict = self._configs.get(platform.lower())
|
||||
if platform_dict is None:
|
||||
return []
|
||||
return sorted(platform_dict.keys(), key=str)
|
||||
|
||||
def to_config_entries(self) -> dict[str, list[dict[str, Any]]]:
|
||||
"""Serialize to config entries format for JSON output."""
|
||||
result: dict[str, list[dict[str, Any]]] = {}
|
||||
for platform, config_dict in self._configs.items():
|
||||
pairs: list[dict[str, Any]] = []
|
||||
for config_key, config in config_dict.items():
|
||||
config_data = json.loads(config.to_json())
|
||||
pairs.append({"key": dict(config_key), "config": config_data})
|
||||
result[platform] = pairs
|
||||
return result
|
||||
|
||||
def to_dict(self) -> dict[str, dict[CaseKey, Any]]:
|
||||
"""Return configs as a nested dict (platform -> key -> config)."""
|
||||
result: dict[str, dict[CaseKey, Any]] = {}
|
||||
for platform, config_dict in self._configs.items():
|
||||
result[platform] = {
|
||||
k: json.loads(v.to_json()) for k, v in config_dict.items()
|
||||
}
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, kernel_name: str, data: dict[str, Any]) -> ConfigSet:
|
||||
config_set = cls(kernel_name)
|
||||
count = 0
|
||||
|
||||
for platform, platform_data in data.items():
|
||||
if platform not in config_set._configs:
|
||||
config_set._configs[platform] = {}
|
||||
|
||||
for entry in platform_data:
|
||||
raw_key = entry["key"]
|
||||
key = CaseKey.default() if not raw_key else CaseKey(raw_key)
|
||||
config = helion.Config(**entry["config"])
|
||||
config_set._configs[platform][key] = config
|
||||
count += 1
|
||||
|
||||
if count > 0:
|
||||
logger.debug(
|
||||
"Loaded %d configs for kernel '%s'",
|
||||
count,
|
||||
kernel_name,
|
||||
)
|
||||
|
||||
return config_set
|
||||
|
||||
def set_config(
|
||||
self,
|
||||
platform: str,
|
||||
config_key: CaseKey,
|
||||
config: helion.Config,
|
||||
) -> None:
|
||||
platform = platform.lower()
|
||||
if platform not in self._configs:
|
||||
self._configs[platform] = {}
|
||||
self._configs[platform][config_key] = config
|
||||
logger.debug(
|
||||
"Set config for kernel '%s': platform='%s', key='%s'",
|
||||
self._kernel_name,
|
||||
platform,
|
||||
config_key,
|
||||
)
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
"""File-level configuration management for Helion kernels (global singleton)."""
|
||||
|
||||
_instance: ConfigManager | None = None
|
||||
_instance_base_dir: Path | None = None
|
||||
|
||||
def __new__(cls, base_dir: str | Path | None = None) -> ConfigManager:
|
||||
resolved_base_dir = cls._resolve_base_dir(base_dir)
|
||||
|
||||
if cls._instance is not None:
|
||||
if cls._instance_base_dir != resolved_base_dir:
|
||||
raise ValueError(
|
||||
f"ConfigManager singleton already exists with base_dir "
|
||||
f"'{cls._instance_base_dir}', cannot create with "
|
||||
f"different base_dir '{resolved_base_dir}'"
|
||||
)
|
||||
return cls._instance
|
||||
|
||||
instance = super().__new__(cls)
|
||||
cls._instance = instance
|
||||
cls._instance_base_dir = resolved_base_dir
|
||||
return instance
|
||||
|
||||
def __init__(self, base_dir: str | Path | None = None):
|
||||
if hasattr(self, "_base_dir"):
|
||||
return
|
||||
|
||||
self._base_dir = self._resolve_base_dir(base_dir)
|
||||
logger.debug("ConfigManager initialized with base_dir: %s", self._base_dir)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_base_dir(base_dir: str | Path | None) -> Path:
|
||||
if base_dir is not None:
|
||||
return Path(base_dir).resolve()
|
||||
return (Path(__file__).parent / "configs").resolve()
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls) -> ConfigManager:
|
||||
if cls._instance is None:
|
||||
raise RuntimeError(
|
||||
"ConfigManager instance has not been created. "
|
||||
"Call ConfigManager(base_dir=...) first to initialize."
|
||||
)
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
def reset_instance(cls) -> None:
|
||||
"""For testing purposes only."""
|
||||
cls._instance = None
|
||||
cls._instance_base_dir = None
|
||||
|
||||
def get_kernel_dir(self, kernel_name: str) -> Path:
|
||||
return self._base_dir / kernel_name
|
||||
|
||||
def get_config_file_path(
|
||||
self, kernel_name: str, platform: str | None = None
|
||||
) -> Path:
|
||||
if platform is not None:
|
||||
return self.get_kernel_dir(kernel_name) / f"{platform}.json"
|
||||
return self.get_kernel_dir(kernel_name)
|
||||
|
||||
def ensure_base_dir_exists(self) -> Path:
|
||||
self._base_dir.mkdir(parents=True, exist_ok=True)
|
||||
return self._base_dir
|
||||
|
||||
def ensure_base_dir_writable(self) -> None:
|
||||
self.ensure_base_dir_exists()
|
||||
test_file = self._base_dir / ".write_test"
|
||||
try:
|
||||
test_file.write_text("test")
|
||||
test_file.unlink()
|
||||
except OSError as e:
|
||||
raise OSError(
|
||||
f"Config directory '{self._base_dir}' is not writable: {e}"
|
||||
) from e
|
||||
|
||||
def _load_platform_file(self, kernel_name: str, platform: str) -> Any:
|
||||
config_path = self.get_config_file_path(kernel_name, platform)
|
||||
if not config_path.exists():
|
||||
return []
|
||||
try:
|
||||
with open(config_path) as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.error("Failed to load config file %s: %s", config_path, e)
|
||||
return []
|
||||
|
||||
def load_config_set(self, kernel_name: str) -> ConfigSet:
|
||||
kernel_dir = self.get_kernel_dir(kernel_name)
|
||||
if not kernel_dir.is_dir():
|
||||
return ConfigSet.from_dict(kernel_name, {})
|
||||
|
||||
data: dict[str, Any] = {}
|
||||
for platform_file in sorted(kernel_dir.glob("*.json")):
|
||||
platform = platform_file.stem
|
||||
try:
|
||||
with open(platform_file) as f:
|
||||
platform_data = json.load(f)
|
||||
data[platform] = platform_data
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.error(
|
||||
"Failed to load config file %s: %s",
|
||||
platform_file,
|
||||
e,
|
||||
)
|
||||
|
||||
return ConfigSet.from_dict(kernel_name, data)
|
||||
|
||||
def get_platform_configs(
|
||||
self, kernel_name: str, platform: str
|
||||
) -> dict[CaseKey, helion.Config]:
|
||||
platform_data = self._load_platform_file(kernel_name, platform)
|
||||
if not platform_data:
|
||||
return {}
|
||||
config_set = ConfigSet.from_dict(kernel_name, {platform: platform_data})
|
||||
return {
|
||||
k: config_set.get_config(platform, k)
|
||||
for k in config_set.get_config_keys(platform)
|
||||
}
|
||||
|
||||
def save_config_set(self, config_set: ConfigSet) -> Path:
|
||||
kernel_dir = self.get_kernel_dir(config_set.kernel_name)
|
||||
kernel_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
full_data = config_set.to_config_entries()
|
||||
for platform, pairs in full_data.items():
|
||||
platform_path = kernel_dir / f"{platform}.json"
|
||||
with open(platform_path, "w") as f:
|
||||
json.dump(pairs, f, indent=2)
|
||||
f.write("\n")
|
||||
logger.info("Saved config to: %s", platform_path)
|
||||
|
||||
return kernel_dir
|
||||
|
||||
def save_configs(
|
||||
self,
|
||||
kernel_name: str,
|
||||
platform: str,
|
||||
configs: dict[CaseKey, helion.Config],
|
||||
) -> Path:
|
||||
"""Save configs for a kernel/platform, merging with existing."""
|
||||
config_set = ConfigSet.from_dict(
|
||||
kernel_name,
|
||||
{platform: self._load_platform_file(kernel_name, platform)},
|
||||
)
|
||||
for key, config in configs.items():
|
||||
config_set.set_config(platform, key, config)
|
||||
|
||||
pairs = config_set.to_config_entries().get(platform, [])
|
||||
platform_path = self.get_config_file_path(kernel_name, platform)
|
||||
platform_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(platform_path, "w") as f:
|
||||
json.dump(pairs, f, indent=2)
|
||||
f.write("\n")
|
||||
|
||||
logger.info("Saved config to: %s", platform_path)
|
||||
return platform_path
|
||||
@@ -1,34 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Helion kernel implementation.
|
||||
|
||||
Importing this package does NOT register any kernels. Runtime code imports the
|
||||
specific op module it needs, e.g.::
|
||||
|
||||
from vllm.kernels.helion.ops import scaled_mm # noqa: F401
|
||||
|
||||
which triggers that op's ``@register_kernel`` as an import side effect.
|
||||
|
||||
Tools that need the full registry (e.g. tools/autotune_helion_kernels.py)
|
||||
call ``import_all_ops()`` to force every op module to register.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import pkgutil
|
||||
|
||||
|
||||
def import_all_kernels() -> list[str]:
|
||||
"""Import every kernel submodule so all ``@register_kernel`` decorators run.
|
||||
|
||||
Returns:
|
||||
The fully-qualified module names that were imported.
|
||||
|
||||
"""
|
||||
imported: list[str] = []
|
||||
for module_info in pkgutil.iter_modules(__path__):
|
||||
if module_info.ispkg:
|
||||
continue
|
||||
module_name = f"{__name__}.{module_info.name}"
|
||||
importlib.import_module(module_name)
|
||||
imported.append(module_name)
|
||||
return imported
|
||||
@@ -1,171 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from itertools import product
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.register import register_kernel
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
raise ImportError(
|
||||
"Helion kernel requires helion to be installed. "
|
||||
"Install it with: pip install helion"
|
||||
)
|
||||
|
||||
import helion
|
||||
import helion.language as hl
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
|
||||
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all input
|
||||
# property combination. Currently, dtypes are fixed. We need optimization to
|
||||
# bucket/skip some combinations
|
||||
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
|
||||
hidden_size_list = [2048, 4096, 5120]
|
||||
in_dtype: torch.dtype = torch.bfloat16
|
||||
out_dtype: torch.dtype = current_platform.fp8_dtype()
|
||||
scale_dtype: torch.dtype = torch.float32
|
||||
inputs = {}
|
||||
for num_tokens, hidden_size in product(num_tokens_list, hidden_size_list):
|
||||
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
|
||||
result = torch.empty(input.shape, device=input.device, dtype=out_dtype)
|
||||
scale = torch.empty((num_tokens, 1), device=input.device, dtype=scale_dtype)
|
||||
# scale_ub clamps the per-token amax of |input|. Use a non-degenerate
|
||||
# upper bound (midway between the mean and max of |input|) so clamping is
|
||||
# partially active and the baseline comparison is meaningful.
|
||||
# torch.mean(input) ~= 0 for the zero-mean input would collapse every
|
||||
# scale to the floor and saturate the output.
|
||||
input_abs = input.to(torch.float32).abs()
|
||||
scale_ub = (0.5 * (input_abs.mean() + input_abs.amax())).to(scale_dtype)
|
||||
|
||||
config_key = CaseKey({"hidden_size": hidden_size, "num_tokens": num_tokens})
|
||||
inputs[config_key] = (result, input, scale, scale_ub)
|
||||
|
||||
return inputs
|
||||
|
||||
|
||||
_pick_cache: dict[tuple[int, int], CaseKey | None] = {}
|
||||
|
||||
|
||||
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
|
||||
"""Pick the best pre-tuned config for the given input shape.
|
||||
|
||||
Selection strategy:
|
||||
1. Find the closest hidden_size among available configs
|
||||
(exact match preferred).
|
||||
2. Among the num_tokens values tuned for that hidden_size, pick
|
||||
the smallest num_tokens >= the input's num_tokens. If the input is
|
||||
larger than all available num_tokens, fall back to the largest.
|
||||
"""
|
||||
if not config_keys:
|
||||
return None
|
||||
|
||||
_, input, *_ = args
|
||||
num_tokens, hidden_size = input.shape
|
||||
|
||||
cache_key = (num_tokens, hidden_size)
|
||||
cached = _pick_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
configs: dict[int, list[int]] = {}
|
||||
for key in config_keys:
|
||||
if key.is_default():
|
||||
continue
|
||||
configs.setdefault(key["hidden_size"], []).append(key["num_tokens"])
|
||||
|
||||
if not configs:
|
||||
return None
|
||||
|
||||
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
|
||||
available_num_tokens = sorted(configs[best_hidden_size])
|
||||
best_num_tokens = next(
|
||||
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
|
||||
)
|
||||
|
||||
result = CaseKey({"hidden_size": best_hidden_size, "num_tokens": best_num_tokens})
|
||||
_pick_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
def baseline(
|
||||
result: torch.Tensor, # [num_tokens, hidden_size]
|
||||
input: torch.Tensor, # [num_tokens, hidden_size]
|
||||
scale: torch.Tensor, # [num_tokens, 1]
|
||||
scale_ub: torch.Tensor | None = None, # scalar tensor
|
||||
) -> None:
|
||||
fp8_min, fp8_max = get_fp8_min_max()
|
||||
min_scaling_factor = 1.0 / (fp8_max * 512.0)
|
||||
|
||||
x = input.to(torch.float32)
|
||||
s = torch.amax(torch.abs(x), dim=-1, keepdim=True)
|
||||
if scale_ub is not None:
|
||||
s = s.clamp(max=scale_ub)
|
||||
s = (s * (1.0 / fp8_max)).clamp(min=min_scaling_factor)
|
||||
y = (x / s).clamp(fp8_min, fp8_max)
|
||||
|
||||
scale.copy_(s)
|
||||
result.copy_(y.to(result.dtype))
|
||||
|
||||
|
||||
# Overwrite autotune_baseline_atol and autotune_baseline_rtol
|
||||
# if too many configs failed due to baseline check during autotuning
|
||||
@register_kernel(
|
||||
mutates_args=["result", "scale"],
|
||||
config_picker=pick_config,
|
||||
input_generator=generate_inputs,
|
||||
helion_settings=helion.Settings(
|
||||
autotune_baseline_fn=baseline,
|
||||
ignore_warnings=[helion.exc.TensorOperationInWrapper],
|
||||
),
|
||||
)
|
||||
def dynamic_per_token_scaled_fp8_quant(
|
||||
result: torch.Tensor, # [num_tokens, hidden_size]
|
||||
input: torch.Tensor, # [num_tokens, hidden_size]
|
||||
scale: torch.Tensor, # [num_tokens, 1]
|
||||
scale_ub: torch.Tensor | None = None, # scalar tensor
|
||||
) -> None:
|
||||
# This code assumes batch_dim and num_tokens are flattened
|
||||
assert input.ndim == 2
|
||||
num_tokens, hidden_size = input.shape
|
||||
hl.specialize(hidden_size)
|
||||
|
||||
assert result.shape == input.shape
|
||||
assert scale.shape[0] == num_tokens
|
||||
assert scale.dtype == torch.float32
|
||||
assert input.stride()[-1] == 1
|
||||
assert result.stride()[-1] == 1
|
||||
|
||||
fp8_min, fp8_max = get_fp8_min_max()
|
||||
min_scaling_factor = 1.0 / (fp8_max * 512.0)
|
||||
|
||||
for tile_m in hl.tile(num_tokens, block_size=1):
|
||||
s_blk = hl.zeros([tile_m], dtype=torch.float32)
|
||||
for tile_n in hl.tile(hidden_size):
|
||||
x_blk = input[tile_m, tile_n].to(dtype=torch.float32)
|
||||
tmp_blk = torch.amax(torch.abs(x_blk), dim=-1)
|
||||
s_blk = torch.maximum(s_blk, tmp_blk)
|
||||
|
||||
if scale_ub is not None:
|
||||
scale_ub_s = hl.load(scale_ub, [])
|
||||
s_blk = s_blk.clamp(max=scale_ub_s)
|
||||
s_blk = s_blk * (1.0 / fp8_max)
|
||||
s_blk = s_blk.clamp(min=min_scaling_factor)
|
||||
scale[tile_m, 0] = s_blk
|
||||
|
||||
for tile_n in hl.tile(hidden_size):
|
||||
x_blk = input[tile_m, tile_n].to(torch.float32)
|
||||
y_blk = x_blk * (1.0 / s_blk[:, None])
|
||||
|
||||
result[tile_m, tile_n] = y_blk.clamp(fp8_min, fp8_max).to(result.dtype)
|
||||
@@ -1,297 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from itertools import product
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
raise ImportError(
|
||||
"Helion kernel requires helion to be installed. "
|
||||
"Install it with: pip install helion"
|
||||
)
|
||||
|
||||
import helion
|
||||
import helion.language as hl
|
||||
|
||||
from vllm import ir
|
||||
from vllm.kernels.helion.register import register_kernel
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _compute_cos_sin_cache(
|
||||
max_position_embeddings, rotary_dim, device="cuda", dtype=torch.float
|
||||
):
|
||||
inv_freq = 1.0 / (
|
||||
10000
|
||||
** (torch.arange(0, rotary_dim, 2, device=device, dtype=dtype) / rotary_dim)
|
||||
)
|
||||
|
||||
t = torch.arange(max_position_embeddings, device=device, dtype=dtype)
|
||||
|
||||
freqs = torch.einsum("i,j -> ij", t, inv_freq)
|
||||
cos = freqs.cos()
|
||||
sin = freqs.sin()
|
||||
cache = torch.cat((cos, sin), dim=-1)
|
||||
return cache
|
||||
|
||||
|
||||
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
|
||||
# TODO(xiaohongchen1991): it is difficult for kernel author to cover
|
||||
# all input property combination. Currently, dtypes are fixed. We need
|
||||
# optimization to bucket/skip some combinations
|
||||
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
|
||||
num_heads_pair = [
|
||||
(16, 8),
|
||||
(32, 8),
|
||||
(64, 8),
|
||||
]
|
||||
head_dim = 128
|
||||
in_dtype: torch.dtype = torch.bfloat16
|
||||
rotary_ratio = 1.0
|
||||
is_neox = True
|
||||
eps = 1e-6
|
||||
device = "cuda"
|
||||
inputs = {}
|
||||
|
||||
for num_tokens, (num_q_heads, num_kv_heads) in product(
|
||||
num_tokens_list, num_heads_pair
|
||||
):
|
||||
total_dim = (num_q_heads + 2 * num_kv_heads) * head_dim
|
||||
qkv = torch.empty(
|
||||
num_tokens, total_dim, dtype=in_dtype, device=device
|
||||
).uniform_(-0.1, 0.1)
|
||||
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
|
||||
q_weight = torch.empty(head_dim, dtype=in_dtype, device=device).uniform_(
|
||||
0.8, 1.2
|
||||
)
|
||||
k_weight = torch.empty(head_dim, dtype=in_dtype, device=device).uniform_(
|
||||
0.8, 1.2
|
||||
)
|
||||
rotary_dim = int(head_dim * rotary_ratio)
|
||||
cos_sin_cache = _compute_cos_sin_cache(40960, rotary_dim)
|
||||
cos_sin_cache = cos_sin_cache.to(in_dtype)
|
||||
|
||||
config_key = CaseKey(
|
||||
{
|
||||
"q_heads": num_q_heads,
|
||||
"kv_heads": num_kv_heads,
|
||||
"num_tokens": num_tokens,
|
||||
}
|
||||
)
|
||||
inputs[config_key] = (
|
||||
qkv,
|
||||
num_q_heads,
|
||||
num_kv_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
eps,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
is_neox,
|
||||
positions.view(-1),
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
|
||||
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
|
||||
|
||||
|
||||
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
|
||||
"""Pick the best pre-tuned config for the given input shape.
|
||||
|
||||
Selection strategy:
|
||||
1. Find the closest q_heads among available configs
|
||||
(exact match preferred).
|
||||
2. Find the closest kv_heads among available configs
|
||||
(exact match preferred).
|
||||
3. Among the num_tokens values tuned for that q_heads and q_heads, pick
|
||||
the smallest num_tokens >= the input's num_tokens. If the input is
|
||||
larger than all available num_tokens, fall back to the largest.
|
||||
"""
|
||||
if not config_keys:
|
||||
return None
|
||||
|
||||
qkv, q_heads, kv_heads, *_ = args
|
||||
num_tokens = qkv.shape[0]
|
||||
|
||||
cache_key = (num_tokens, q_heads, kv_heads)
|
||||
cached = _pick_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
configs: dict[int, dict[int, list[int]]] = {}
|
||||
for key in config_keys:
|
||||
if key.is_default():
|
||||
continue
|
||||
configs.setdefault(key["q_heads"], {}).setdefault(key["kv_heads"], []).append(
|
||||
key["num_tokens"]
|
||||
)
|
||||
|
||||
if not configs:
|
||||
return None
|
||||
|
||||
best_q_heads = min(configs, key=lambda s: abs(s - q_heads))
|
||||
best_kv_heads = min(configs[best_q_heads], key=lambda s: abs(s - kv_heads))
|
||||
available_num_tokens = sorted(configs[best_q_heads][best_kv_heads])
|
||||
best_num_tokens = next(
|
||||
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
|
||||
)
|
||||
|
||||
result = CaseKey(
|
||||
{
|
||||
"q_heads": best_q_heads,
|
||||
"kv_heads": best_kv_heads,
|
||||
"num_tokens": best_num_tokens,
|
||||
}
|
||||
)
|
||||
_pick_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
def baseline(
|
||||
qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]
|
||||
num_heads_q: int,
|
||||
num_heads_k: int,
|
||||
num_heads_v: int,
|
||||
head_dim: int,
|
||||
eps: float,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor, # [max_position, rotary_dim]
|
||||
is_neox: bool,
|
||||
position_ids: torch.Tensor, # [num_tokens],
|
||||
forced_token_heads_per_warp: int = -1, # dummy
|
||||
) -> None:
|
||||
q_size = num_heads_q * head_dim
|
||||
kv_size = num_heads_k * head_dim
|
||||
|
||||
q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
|
||||
|
||||
q_by_head = q.view(*q.shape[:-1], q.shape[-1] // head_dim, head_dim)
|
||||
q_by_head = ir.ops.rms_norm(q_by_head, q_weight, eps)
|
||||
q = q_by_head.view(q.shape)
|
||||
|
||||
k_by_head = k.view(*k.shape[:-1], k.shape[-1] // head_dim, head_dim)
|
||||
k_by_head = ir.ops.rms_norm(k_by_head, k_weight, eps)
|
||||
k = k_by_head.view(k.shape)
|
||||
|
||||
q, k = RotaryEmbedding.forward_static(
|
||||
position_ids, q, k, head_dim, cos_sin_cache.shape[1], cos_sin_cache, is_neox
|
||||
)
|
||||
qkv[:, :q_size].copy_(q)
|
||||
qkv[:, q_size : q_size + kv_size].copy_(k)
|
||||
|
||||
|
||||
# Overwrite autotune_baseline_atol and autotune_baseline_rtol
|
||||
# if too many configs failed due to baseline check during autotuning
|
||||
@register_kernel(
|
||||
mutates_args=["qkv"],
|
||||
config_picker=pick_config,
|
||||
input_generator=generate_inputs,
|
||||
helion_settings=helion.Settings(
|
||||
autotune_baseline_fn=baseline,
|
||||
autotune_baseline_atol=5e-2,
|
||||
autotune_baseline_rtol=5e-2,
|
||||
ignore_warnings=[helion.exc.TensorOperationInWrapper],
|
||||
),
|
||||
) # type: ignore[misc]
|
||||
def fused_qk_norm_rope(
|
||||
qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]
|
||||
num_heads_q: int,
|
||||
num_heads_k: int,
|
||||
num_heads_v: int,
|
||||
head_dim: int,
|
||||
eps: float,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor, # [max_position, rotary_dim]
|
||||
is_neox: bool,
|
||||
position_ids: torch.Tensor, # [num_tokens],
|
||||
forced_token_heads_per_warp: int = -1, # dummy
|
||||
) -> None:
|
||||
assert qkv.ndim == 2
|
||||
num_tokens = qkv.shape[0]
|
||||
total_heads = num_heads_q + num_heads_k + num_heads_v
|
||||
assert qkv.shape[1] == total_heads * head_dim
|
||||
hl.specialize(qkv.shape[1])
|
||||
|
||||
assert cos_sin_cache.ndim == 2
|
||||
max_position, rotary_dim = cos_sin_cache.shape
|
||||
hl.specialize(max_position)
|
||||
hl.specialize(rotary_dim)
|
||||
assert rotary_dim % 2 == 0
|
||||
assert rotary_dim <= head_dim
|
||||
embed_dim = rotary_dim // 2
|
||||
|
||||
hl.specialize(num_heads_q)
|
||||
hl.specialize(num_heads_k)
|
||||
hl.specialize(num_heads_v)
|
||||
hl.specialize(head_dim)
|
||||
|
||||
assert position_ids.ndim == 1 and position_ids.shape[0] == num_tokens
|
||||
hl.specialize(position_ids.shape[0])
|
||||
|
||||
assert q_weight.ndim == 1 and q_weight.shape[0] == head_dim
|
||||
hl.specialize(q_weight.shape[0])
|
||||
assert k_weight.ndim == 1 and k_weight.shape[0] == head_dim
|
||||
hl.specialize(k_weight.shape[0])
|
||||
|
||||
assert qkv.dtype == q_weight.dtype and q_weight.dtype == k_weight.dtype
|
||||
assert position_ids.dtype == torch.int64
|
||||
|
||||
assert qkv.is_contiguous()
|
||||
assert position_ids.is_contiguous()
|
||||
assert q_weight.is_contiguous()
|
||||
assert k_weight.is_contiguous()
|
||||
assert cos_sin_cache.is_contiguous()
|
||||
|
||||
qk_heads = num_heads_q + num_heads_k
|
||||
|
||||
qkv = qkv.view(num_tokens, -1, head_dim)
|
||||
|
||||
for tile_m, tile_gn, tile_n in hl.tile(
|
||||
[num_tokens, qk_heads, head_dim], block_size=[1, None, head_dim]
|
||||
):
|
||||
x_blk = qkv[tile_m, tile_gn, tile_n].to(dtype=torch.float32)
|
||||
|
||||
rms = x_blk.pow(2).sum(dim=-1)
|
||||
rms = torch.rsqrt(rms * (1.0 / head_dim) + eps)
|
||||
|
||||
use_q_weight = (tile_gn.index < num_heads_q)[None, :, None]
|
||||
w_blk = torch.where(
|
||||
use_q_weight, q_weight[None, None, tile_n], k_weight[None, None, tile_n]
|
||||
)
|
||||
|
||||
x_blk = (x_blk * rms[:, :, None]).to(qkv.dtype) * w_blk
|
||||
|
||||
qkv[tile_m, tile_gn, tile_n] = x_blk
|
||||
|
||||
pos_id = position_ids[tile_m]
|
||||
cos_blk = cos_sin_cache[pos_id, hl.arange(embed_dim)]
|
||||
sin_blk = cos_sin_cache[pos_id, hl.arange(embed_dim) + embed_dim]
|
||||
|
||||
if is_neox:
|
||||
x1_offset = hl.arange(embed_dim)
|
||||
x2_offset = x1_offset + embed_dim
|
||||
else:
|
||||
x1_offset = hl.arange(embed_dim) * 2
|
||||
x2_offset = x1_offset + 1
|
||||
|
||||
x1_blk = qkv[tile_m, tile_gn, x1_offset]
|
||||
x2_blk = qkv[tile_m, tile_gn, x2_offset]
|
||||
|
||||
o1_blk = x1_blk * cos_blk[:, None, :] - x2_blk * sin_blk[:, None, :]
|
||||
o2_blk = x2_blk * cos_blk[:, None, :] + x1_blk * sin_blk[:, None, :]
|
||||
|
||||
qkv[tile_m, tile_gn, x1_offset] = o1_blk
|
||||
qkv[tile_m, tile_gn, x2_offset] = o2_blk
|
||||
@@ -1,212 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from itertools import product
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
raise ImportError(
|
||||
"Helion kernel requires helion to be installed. "
|
||||
"Install it with: pip install helion"
|
||||
)
|
||||
|
||||
import helion
|
||||
import helion.language as hl
|
||||
|
||||
from vllm.kernels.helion.register import register_kernel
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
|
||||
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all
|
||||
# input property combination. Currently, dtypes are fixed. We need
|
||||
# optimization to bucket/skip some combinations
|
||||
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
|
||||
hidden_size_list = [2048, 4096, 5120]
|
||||
group_size_list = [128]
|
||||
in_dtype: torch.dtype = torch.bfloat16
|
||||
out_dtype: torch.dtype = current_platform.fp8_dtype()
|
||||
scale_dtype: torch.dtype = torch.float32
|
||||
|
||||
use_ue8m0 = False
|
||||
column_major = False
|
||||
fp8_min, fp8_max = get_fp8_min_max()
|
||||
eps = 1e-10
|
||||
|
||||
inputs = {}
|
||||
|
||||
for hidden_size, group_size, num_tokens in product(
|
||||
hidden_size_list, group_size_list, num_tokens_list
|
||||
):
|
||||
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
|
||||
output_q = torch.empty(input.shape, device=input.device, dtype=out_dtype)
|
||||
output_s = torch.empty(
|
||||
(num_tokens, hidden_size // group_size),
|
||||
device=input.device,
|
||||
dtype=scale_dtype,
|
||||
)
|
||||
config_key = CaseKey(
|
||||
{
|
||||
"hidden_size": hidden_size,
|
||||
"group_size": group_size,
|
||||
"num_tokens": num_tokens,
|
||||
}
|
||||
)
|
||||
inputs[config_key] = (
|
||||
input,
|
||||
output_q,
|
||||
output_s,
|
||||
group_size,
|
||||
eps,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
use_ue8m0,
|
||||
column_major,
|
||||
False,
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
|
||||
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
|
||||
|
||||
|
||||
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
|
||||
"""Pick the best pre-tuned config for the given input shape.
|
||||
|
||||
Selection strategy:
|
||||
1. Find the closest hidden_size among available configs
|
||||
(exact match preferred).
|
||||
2. Find the closest group_size among available configs
|
||||
(exact match preferred).
|
||||
3. Among the num_tokens values tuned for that hidden_size and group_size, pick
|
||||
the smallest num_tokens >= the input's num_tokens. If the input is
|
||||
larger than all available num_tokens, fall back to the largest.
|
||||
"""
|
||||
if not config_keys:
|
||||
return None
|
||||
|
||||
input, _, _, group_size, *_ = args
|
||||
num_tokens, hidden_size = input.shape
|
||||
|
||||
cache_key = (num_tokens, group_size, hidden_size)
|
||||
cached = _pick_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
configs: dict[int, dict[int, list[int]]] = {}
|
||||
for key in config_keys:
|
||||
if key.is_default():
|
||||
continue
|
||||
configs.setdefault(key["hidden_size"], {}).setdefault(
|
||||
key["group_size"], []
|
||||
).append(key["num_tokens"])
|
||||
|
||||
if not configs:
|
||||
return None
|
||||
|
||||
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
|
||||
best_group_size = min(configs[best_hidden_size], key=lambda s: abs(s - group_size))
|
||||
available_num_tokens = sorted(configs[best_hidden_size][best_group_size])
|
||||
best_num_tokens = next(
|
||||
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
|
||||
)
|
||||
|
||||
result = CaseKey(
|
||||
{
|
||||
"hidden_size": best_hidden_size,
|
||||
"group_size": best_group_size,
|
||||
"num_tokens": best_num_tokens,
|
||||
}
|
||||
)
|
||||
_pick_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
def baseline(
|
||||
input: torch.Tensor, # [num_tokens, hidden_size]
|
||||
output_q: torch.Tensor, # [num_tokens, hidden_size]
|
||||
output_s: torch.Tensor, # [num_tokens, groups_per_row]
|
||||
group_size: int,
|
||||
eps: float,
|
||||
fp8_min: float,
|
||||
fp8_max: float,
|
||||
scale_ue8m0: bool,
|
||||
dummy_is_scale_transposed: bool = False,
|
||||
dummy_is_tma_aligned: bool = False,
|
||||
) -> None:
|
||||
num_tokens, hidden_size = input.shape
|
||||
groups_per_row = hidden_size // group_size
|
||||
|
||||
x = input.view(num_tokens, groups_per_row, group_size).to(torch.float32)
|
||||
s = torch.clamp(torch.amax(torch.abs(x), dim=-1), min=eps) / fp8_max
|
||||
if scale_ue8m0:
|
||||
s = torch.exp2(torch.ceil(torch.log2(s)))
|
||||
y = torch.clamp(x / s[:, :, None], fp8_min, fp8_max)
|
||||
|
||||
output_s.copy_(s)
|
||||
output_q.copy_(y.view(num_tokens, hidden_size).to(output_q.dtype))
|
||||
|
||||
|
||||
@register_kernel(
|
||||
mutates_args=["output_q", "output_s"],
|
||||
config_picker=pick_config,
|
||||
input_generator=generate_inputs,
|
||||
helion_settings=helion.Settings(
|
||||
autotune_baseline_fn=baseline,
|
||||
),
|
||||
) # type: ignore[misc]
|
||||
def per_token_group_fp8_quant(
|
||||
input: torch.Tensor, # [num_tokens, hidden_size]
|
||||
output_q: torch.Tensor, # [num_tokens, hidden_size]
|
||||
output_s: torch.Tensor, # [num_tokens, groups_per_row]
|
||||
group_size: int,
|
||||
eps: float,
|
||||
fp8_min: float,
|
||||
fp8_max: float,
|
||||
scale_ue8m0: bool,
|
||||
# Unused dummy args
|
||||
# Kept for consistency with existing kernel interface
|
||||
dummy_is_scale_transposed: bool = False,
|
||||
dummy_is_tma_aligned: bool = False,
|
||||
) -> None:
|
||||
# This code assumes batch_dim and num_tokens are flattened
|
||||
assert input.ndim == 2
|
||||
num_tokens, hidden_size = input.shape
|
||||
hl.specialize(hidden_size)
|
||||
hl.specialize(group_size)
|
||||
|
||||
groups_per_row = output_s.shape[1]
|
||||
hl.specialize(groups_per_row)
|
||||
assert hidden_size % group_size == 0 and hidden_size // group_size == groups_per_row
|
||||
assert output_s.ndim == 2 and output_s.dtype == torch.float32
|
||||
|
||||
input = input.view(num_tokens, -1, group_size)
|
||||
output_q = output_q.view(num_tokens, -1, group_size)
|
||||
for tile_m, tile_gn, tile_n in hl.tile(
|
||||
[num_tokens, groups_per_row, group_size], block_size=[1, None, group_size]
|
||||
):
|
||||
x_blk = input[tile_m, tile_gn, tile_n]
|
||||
y_s_blk = torch.clamp(torch.amax(torch.abs(x_blk), dim=-1), min=eps)
|
||||
y_s_blk = y_s_blk / fp8_max
|
||||
|
||||
if scale_ue8m0:
|
||||
y_s_blk = torch.exp2(torch.ceil(torch.log2(y_s_blk)))
|
||||
|
||||
y_q_blk = torch.clamp(x_blk / y_s_blk[:, :, None], fp8_min, fp8_max).to(
|
||||
output_q.dtype
|
||||
)
|
||||
|
||||
output_s[tile_m, tile_gn] = y_s_blk
|
||||
output_q[tile_m, tile_gn, tile_n] = y_q_blk
|
||||
@@ -1,256 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from itertools import product
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.utils import (
|
||||
get_fp8_dtype,
|
||||
get_int8_min_max,
|
||||
get_int8_min_scaling_factor,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
raise ImportError(
|
||||
"Helion kernel requires helion to be installed. "
|
||||
"Install it with: pip install helion"
|
||||
)
|
||||
|
||||
import helion
|
||||
import helion.language as hl
|
||||
|
||||
from vllm.kernels.helion.register import register_kernel
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
|
||||
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all
|
||||
# input property combination. Currently, dtypes are fixed. We need
|
||||
# optimization to bucket/skip some combinations
|
||||
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
|
||||
hidden_size_list = [2048, 4096, 5120]
|
||||
in_dtype: torch.dtype = torch.bfloat16
|
||||
out_dtype: torch.dtype = current_platform.fp8_dtype()
|
||||
scale_dtype: torch.dtype = torch.float32
|
||||
inputs = {}
|
||||
|
||||
for num_tokens, hidden_size in product(num_tokens_list, hidden_size_list):
|
||||
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
|
||||
result = torch.empty(input.shape, device=input.device, dtype=out_dtype)
|
||||
scale = torch.empty((num_tokens, 1), device=input.device, dtype=scale_dtype)
|
||||
residual = torch.randn_like(input)
|
||||
weight = torch.normal(
|
||||
mean=1.0,
|
||||
std=1.0,
|
||||
size=(hidden_size,),
|
||||
dtype=input.dtype,
|
||||
device=input.device,
|
||||
)
|
||||
epsilon = 1e-6
|
||||
# scale_ub clamps the per-token amax of the RMS-normed, weighted output.
|
||||
# Use a non-degenerate upper bound (midway between the mean and max of
|
||||
# that magnitude) so clamping is partially active and the baseline
|
||||
# comparison is meaningful. torch.mean(input) ~= 0 for the zero-mean
|
||||
# input would collapse every scale to the floor and saturate the output.
|
||||
# Mirrors the reference normalization in baseline() below.
|
||||
x = input.to(torch.float32) + residual.to(torch.float32)
|
||||
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
|
||||
x_norm_abs = ((x * rms).to(input.dtype) * weight).abs().to(torch.float32)
|
||||
scale_ub = (0.5 * (x_norm_abs.mean() + x_norm_abs.amax())).to(scale_dtype)
|
||||
|
||||
config_key = CaseKey({"hidden_size": hidden_size, "num_tokens": num_tokens})
|
||||
inputs[config_key] = (result, input, weight, scale, epsilon, scale_ub, residual)
|
||||
|
||||
return inputs
|
||||
|
||||
|
||||
_pick_cache: dict[tuple[int, int], CaseKey | None] = {}
|
||||
|
||||
|
||||
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
|
||||
"""Pick the best pre-tuned config for the given input shape.
|
||||
|
||||
Selection strategy:
|
||||
1. Find the closest hidden_size among available configs
|
||||
(exact match preferred).
|
||||
2. Among the num_tokens values tuned for that hidden_size, pick
|
||||
the smallest num_tokens >= the input's num_tokens. If the input is
|
||||
larger than all available num_tokens, fall back to the largest.
|
||||
"""
|
||||
if not config_keys:
|
||||
return None
|
||||
|
||||
_, input, *_ = args
|
||||
num_tokens, hidden_size = input.shape
|
||||
|
||||
cache_key = (num_tokens, hidden_size)
|
||||
cached = _pick_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
configs: dict[int, list[int]] = {}
|
||||
for key in config_keys:
|
||||
if key.is_default():
|
||||
continue
|
||||
configs.setdefault(key["hidden_size"], []).append(key["num_tokens"])
|
||||
|
||||
if not configs:
|
||||
return None
|
||||
|
||||
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
|
||||
available_num_tokens = sorted(configs[best_hidden_size])
|
||||
best_num_tokens = next(
|
||||
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
|
||||
)
|
||||
|
||||
result = CaseKey({"hidden_size": best_hidden_size, "num_tokens": best_num_tokens})
|
||||
_pick_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
def baseline(
|
||||
result: torch.Tensor, # [num_tokens, hidden_size]
|
||||
input: torch.Tensor, # [num_tokens, hidden_size]
|
||||
weight: torch.Tensor, # [num_tokens]
|
||||
scale: torch.Tensor, # [num_tokens, 1]
|
||||
epsilon: float,
|
||||
scale_ub: torch.Tensor | None = None, # []
|
||||
residual: torch.Tensor | None = None, # [num_tokens, hidden_size]
|
||||
) -> None:
|
||||
_, hidden_size = input.shape
|
||||
quant_dtype = result.dtype
|
||||
qtype_min: int | float
|
||||
qtype_max: int | float
|
||||
|
||||
if quant_dtype == torch.int8:
|
||||
qtype_min, qtype_max = get_int8_min_max()
|
||||
min_scaling_factor = get_int8_min_scaling_factor()
|
||||
else:
|
||||
qtype_min, qtype_max = get_fp8_min_max()
|
||||
min_scaling_factor = 1.0 / (qtype_max * 512.0)
|
||||
|
||||
x = input.to(torch.float32)
|
||||
if residual is not None:
|
||||
x = x + residual
|
||||
residual.copy_(x.to(residual.dtype))
|
||||
|
||||
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
|
||||
x_norm = (x * rms).to(input.dtype) * weight
|
||||
|
||||
s = torch.amax(torch.abs(x_norm), dim=-1, keepdim=True).to(torch.float32)
|
||||
if scale_ub is not None:
|
||||
s = s.clamp(max=scale_ub)
|
||||
s = (s * (1.0 / qtype_max)).clamp(min=min_scaling_factor)
|
||||
|
||||
y = x_norm / s
|
||||
if quant_dtype == torch.int8:
|
||||
y = y.round()
|
||||
|
||||
scale.copy_(s)
|
||||
result.copy_(y.clamp(qtype_min, qtype_max).to(result.dtype))
|
||||
|
||||
|
||||
# Overwrite autotune_baseline_atol and autotune_baseline_rtol
|
||||
# if too many configs failed due to baseline check during autotuning
|
||||
@register_kernel(
|
||||
mutates_args=["result", "scale", "residual"],
|
||||
config_picker=pick_config,
|
||||
input_generator=generate_inputs,
|
||||
helion_settings=helion.Settings(
|
||||
autotune_baseline_fn=baseline,
|
||||
ignore_warnings=[helion.exc.TensorOperationInWrapper],
|
||||
),
|
||||
) # type: ignore[misc]
|
||||
def rms_norm_dynamic_per_token_quant(
|
||||
result: torch.Tensor, # [num_tokens, hidden_size]
|
||||
input: torch.Tensor, # [num_tokens, hidden_size]
|
||||
weight: torch.Tensor, # [hidden_size]
|
||||
scale: torch.Tensor, # [num_tokens, 1]
|
||||
epsilon: float,
|
||||
scale_ub: torch.Tensor | None = None, # []
|
||||
residual: torch.Tensor | None = None, # [num_tokens, hidden_size]
|
||||
) -> None:
|
||||
# This code assumes batch_dim and num_tokens are flattened
|
||||
assert input.ndim == 2
|
||||
num_tokens, hidden_size = input.shape
|
||||
hl.specialize(hidden_size)
|
||||
|
||||
fp8_dtype = get_fp8_dtype()
|
||||
assert result.dtype in [fp8_dtype, torch.int8]
|
||||
assert result.is_contiguous() and input.is_contiguous()
|
||||
|
||||
if scale_ub is not None:
|
||||
assert result.dtype == fp8_dtype
|
||||
assert scale_ub.dtype == torch.float32
|
||||
|
||||
assert input.dtype == weight.dtype
|
||||
assert scale.shape[0] == num_tokens
|
||||
assert scale.dtype == torch.float32
|
||||
|
||||
if residual is not None:
|
||||
assert residual.dtype == input.dtype
|
||||
|
||||
quant_dtype = result.dtype
|
||||
qtype_traits_min: int | float
|
||||
qtype_traits_max: int | float
|
||||
if quant_dtype == torch.int8:
|
||||
qtype_traits_min, qtype_traits_max = get_int8_min_max()
|
||||
min_scaling_factor = get_int8_min_scaling_factor()
|
||||
else:
|
||||
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
|
||||
min_scaling_factor = 1.0 / (qtype_traits_max * 512.0)
|
||||
|
||||
qtype_max = float(qtype_traits_max)
|
||||
|
||||
for tile_m in hl.tile(num_tokens, block_size=1):
|
||||
rms = hl.zeros([tile_m], dtype=torch.float32)
|
||||
for tile_n in hl.tile(hidden_size):
|
||||
x_blk = input[tile_m, tile_n].to(torch.float32)
|
||||
if residual is not None:
|
||||
x_blk = x_blk + residual[tile_m, tile_n]
|
||||
rms = rms + x_blk.pow(2).sum(dim=-1)
|
||||
|
||||
rms = torch.rsqrt(rms * (1.0 / hidden_size) + epsilon)
|
||||
s_blk = hl.zeros([tile_m], dtype=torch.float32)
|
||||
|
||||
for tile_n in hl.tile(hidden_size):
|
||||
x_blk = input[tile_m, tile_n].to(torch.float32)
|
||||
if residual is not None:
|
||||
x_blk = x_blk + residual[tile_m, tile_n]
|
||||
x_blk = (x_blk * rms[:, None]).to(input.dtype) * weight[None, tile_n]
|
||||
tmp_blk = torch.amax(torch.abs(x_blk), dim=-1).to(torch.float32)
|
||||
s_blk = torch.maximum(s_blk, tmp_blk)
|
||||
|
||||
if scale_ub is not None:
|
||||
scale_ub_s = hl.load(scale_ub, [])
|
||||
s_blk = s_blk.clamp(max=scale_ub_s)
|
||||
s_blk = s_blk * (1.0 / qtype_max)
|
||||
s_blk = s_blk.clamp(min=min_scaling_factor)
|
||||
scale[tile_m, 0] = s_blk
|
||||
|
||||
for tile_n in hl.tile(hidden_size):
|
||||
x_blk = input[tile_m, tile_n].to(torch.float32)
|
||||
if residual is not None:
|
||||
x_blk = x_blk + residual[tile_m, tile_n]
|
||||
residual[tile_m, tile_n] = x_blk.to(residual.dtype)
|
||||
x_blk = (x_blk * rms[:, None]).to(input.dtype) * weight[None, tile_n]
|
||||
if quant_dtype == torch.int8:
|
||||
s_inv_blk = 1.0 / s_blk[:, None]
|
||||
y_blk = x_blk * s_inv_blk
|
||||
y_blk = y_blk.round()
|
||||
else:
|
||||
y_blk = x_blk / s_blk[:, None]
|
||||
|
||||
result[tile_m, tile_n] = y_blk.clamp(qtype_traits_min, qtype_traits_max).to(
|
||||
result.dtype
|
||||
)
|
||||
@@ -1,316 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from itertools import product
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.utils import (
|
||||
get_fp8_dtype,
|
||||
get_int8_min_max,
|
||||
get_int8_min_scaling_factor,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
raise ImportError(
|
||||
"Helion kernel requires helion to be installed. "
|
||||
"Install it with: pip install helion"
|
||||
)
|
||||
|
||||
import helion
|
||||
import helion.language as hl
|
||||
|
||||
from vllm.kernels.helion.register import register_kernel
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
|
||||
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all
|
||||
# input property combination. Currently, dtypes are fixed. We need
|
||||
# optimization to bucket/skip some combinations
|
||||
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
|
||||
hidden_size_list = [2048, 4096, 5120]
|
||||
group_size_list = [128]
|
||||
in_dtype: torch.dtype = torch.bfloat16
|
||||
out_dtype: torch.dtype = current_platform.fp8_dtype()
|
||||
scale_dtype: torch.dtype = torch.float32
|
||||
inputs = {}
|
||||
|
||||
for hidden_size, group_size, num_tokens in product(
|
||||
hidden_size_list, group_size_list, num_tokens_list
|
||||
):
|
||||
input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype)
|
||||
result = torch.empty(input.shape, device=input.device, dtype=out_dtype)
|
||||
scale = torch.empty(
|
||||
(num_tokens, hidden_size // group_size),
|
||||
device=input.device,
|
||||
dtype=scale_dtype,
|
||||
)
|
||||
residual = torch.randn_like(input)
|
||||
weight = torch.normal(
|
||||
mean=1.0,
|
||||
std=1.0,
|
||||
size=(hidden_size,),
|
||||
dtype=input.dtype,
|
||||
device=input.device,
|
||||
)
|
||||
epsilon = 1e-6
|
||||
# scale_ub clamps the per-group amax of the RMS-normed, weighted output.
|
||||
# Use a non-degenerate upper bound (midway between the mean and max of
|
||||
# that magnitude) so clamping is partially active and the baseline
|
||||
# comparison is meaningful. torch.mean(input) ~= 0 for the zero-mean
|
||||
# input would collapse every scale to the floor and saturate the output.
|
||||
# Mirrors the reference normalization in baseline() below.
|
||||
x = input.to(torch.float32) + residual.to(torch.float32)
|
||||
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
|
||||
x_norm_abs = ((x * rms).to(input.dtype) * weight).abs().to(torch.float32)
|
||||
scale_ub = (0.5 * (x_norm_abs.mean() + x_norm_abs.amax())).to(scale_dtype)
|
||||
|
||||
config_key = CaseKey(
|
||||
{
|
||||
"hidden_size": hidden_size,
|
||||
"group_size": group_size,
|
||||
"num_tokens": num_tokens,
|
||||
}
|
||||
)
|
||||
inputs[config_key] = (
|
||||
result,
|
||||
input,
|
||||
weight,
|
||||
scale,
|
||||
epsilon,
|
||||
scale_ub,
|
||||
residual,
|
||||
group_size,
|
||||
False,
|
||||
)
|
||||
|
||||
return inputs
|
||||
|
||||
|
||||
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
|
||||
|
||||
|
||||
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
|
||||
"""Pick the best pre-tuned config for the given input shape.
|
||||
|
||||
Selection strategy:
|
||||
1. Find the closest hidden_size among available configs
|
||||
(exact match preferred).
|
||||
2. Find the closest group_size among available configs
|
||||
(exact match preferred).
|
||||
2. Among the num_tokens values tuned for that hidden_size and group_size, pick
|
||||
the smallest num_tokens >= the input's num_tokens. If the input is
|
||||
larger than all available num_tokens, fall back to the largest.
|
||||
"""
|
||||
if not config_keys:
|
||||
return None
|
||||
|
||||
_, input, _, _, _, _, _, group_size, *_ = args
|
||||
num_tokens, hidden_size = input.shape
|
||||
|
||||
cache_key = (num_tokens, group_size, hidden_size)
|
||||
cached = _pick_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
configs: dict[int, dict[int, list[int]]] = {}
|
||||
for key in config_keys:
|
||||
if key.is_default():
|
||||
continue
|
||||
configs.setdefault(key["hidden_size"], {}).setdefault(
|
||||
key["group_size"], []
|
||||
).append(key["num_tokens"])
|
||||
|
||||
if not configs:
|
||||
return None
|
||||
|
||||
best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size))
|
||||
best_group_size = min(configs[best_hidden_size], key=lambda s: abs(s - group_size))
|
||||
available_num_tokens = sorted(configs[best_hidden_size][best_group_size])
|
||||
best_num_tokens = next(
|
||||
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
|
||||
)
|
||||
|
||||
result = CaseKey(
|
||||
{
|
||||
"hidden_size": best_hidden_size,
|
||||
"group_size": best_group_size,
|
||||
"num_tokens": best_num_tokens,
|
||||
}
|
||||
)
|
||||
_pick_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
def baseline(
|
||||
result: torch.Tensor, # [num_tokens, hidden_size]
|
||||
input: torch.Tensor, # [num_tokens, hidden_size]
|
||||
weight: torch.Tensor, # [hidden_size]
|
||||
scale: torch.Tensor, # [num_tokens, groups_per_row]
|
||||
epsilon: float,
|
||||
scale_ub: torch.Tensor | None, # []
|
||||
residual: torch.Tensor | None, # [num_tokens, hidden_size]
|
||||
group_size: int,
|
||||
is_scale_transposed: bool,
|
||||
) -> None:
|
||||
num_tokens, hidden_size = input.shape
|
||||
groups_per_row = hidden_size // group_size
|
||||
quant_dtype = result.dtype
|
||||
qtype_min: int | float
|
||||
qtype_max: int | float
|
||||
|
||||
if quant_dtype == torch.int8:
|
||||
qtype_min, qtype_max = get_int8_min_max()
|
||||
min_scaling_factor = get_int8_min_scaling_factor()
|
||||
else:
|
||||
qtype_min, qtype_max = get_fp8_min_max()
|
||||
min_scaling_factor = 1.0 / (qtype_max * 512.0)
|
||||
|
||||
x = input.to(torch.float32)
|
||||
if residual is not None:
|
||||
x = x + residual
|
||||
residual.copy_(x.to(residual.dtype))
|
||||
|
||||
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + epsilon)
|
||||
x_norm = (x * rms).to(input.dtype) * weight
|
||||
x_grouped = x_norm.view(num_tokens, groups_per_row, group_size).to(torch.float32)
|
||||
|
||||
s = torch.amax(torch.abs(x_grouped), dim=-1).to(torch.float32)
|
||||
if scale_ub is not None:
|
||||
s = s.clamp(max=scale_ub)
|
||||
s = (s * (1.0 / qtype_max)).clamp(min=min_scaling_factor)
|
||||
|
||||
y = x_grouped / s[:, :, None]
|
||||
if quant_dtype == torch.int8:
|
||||
y = y.round()
|
||||
|
||||
scale.copy_(s)
|
||||
result.copy_(
|
||||
y.clamp(qtype_min, qtype_max).view(num_tokens, hidden_size).to(result.dtype)
|
||||
)
|
||||
|
||||
|
||||
@register_kernel(
|
||||
mutates_args=["result", "scale", "residual"],
|
||||
config_picker=pick_config,
|
||||
input_generator=generate_inputs,
|
||||
helion_settings=helion.Settings(
|
||||
autotune_baseline_fn=baseline,
|
||||
ignore_warnings=[helion.exc.TensorOperationInWrapper],
|
||||
),
|
||||
) # type: ignore[misc]
|
||||
def rms_norm_per_block_quant(
|
||||
result: torch.Tensor, # [num_tokens, hidden_size]
|
||||
input: torch.Tensor, # [num_tokens, hidden_size]
|
||||
weight: torch.Tensor, # [hidden_size]
|
||||
scale: torch.Tensor, # [num_tokens, groups_per_row]
|
||||
epsilon: float,
|
||||
scale_ub: torch.Tensor | None, # []
|
||||
residual: torch.Tensor | None, # [num_tokens, hidden_size]
|
||||
group_size: int,
|
||||
is_scale_transposed: bool, # dummy
|
||||
) -> None:
|
||||
# This code assumes batch_dim and num_tokens are flattened
|
||||
assert input.ndim == 2
|
||||
num_tokens, hidden_size = input.shape
|
||||
hl.specialize(hidden_size)
|
||||
hl.specialize(group_size)
|
||||
|
||||
groups_per_row = scale.shape[1]
|
||||
hl.specialize(groups_per_row)
|
||||
assert hidden_size % group_size == 0 and hidden_size // group_size == groups_per_row
|
||||
assert scale.shape[0] == num_tokens
|
||||
assert scale.dtype == torch.float32
|
||||
if scale.stride(1) > 1:
|
||||
assert is_scale_transposed
|
||||
|
||||
fp8_dtype = get_fp8_dtype()
|
||||
assert result.dtype in [fp8_dtype, torch.int8]
|
||||
assert result.is_contiguous() and input.is_contiguous()
|
||||
|
||||
if scale_ub is not None:
|
||||
assert result.dtype == fp8_dtype
|
||||
assert scale_ub.dtype == torch.float32
|
||||
|
||||
assert input.dtype == weight.dtype
|
||||
|
||||
if residual is not None:
|
||||
assert residual.dtype == input.dtype
|
||||
|
||||
assert group_size in [64, 128]
|
||||
|
||||
quant_dtype = result.dtype
|
||||
qtype_traits_min: int | float
|
||||
qtype_traits_max: int | float
|
||||
if quant_dtype == torch.int8:
|
||||
qtype_traits_min, qtype_traits_max = get_int8_min_max()
|
||||
min_scaling_factor = get_int8_min_scaling_factor()
|
||||
else:
|
||||
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
|
||||
min_scaling_factor = 1.0 / (qtype_traits_max * 512.0)
|
||||
|
||||
qtype_max = float(qtype_traits_max)
|
||||
|
||||
for tile_m in hl.tile(num_tokens, block_size=1):
|
||||
rms = hl.zeros([tile_m], dtype=torch.float32)
|
||||
for tile_n in hl.tile(hidden_size):
|
||||
x_blk = input[tile_m, tile_n].to(torch.float32)
|
||||
if residual is not None:
|
||||
x_blk = x_blk + residual[tile_m, tile_n]
|
||||
rms = rms + x_blk.pow(2).sum(dim=-1)
|
||||
|
||||
rms = torch.rsqrt(rms * (1.0 / hidden_size) + epsilon)
|
||||
|
||||
m_idx = tile_m.begin + hl.arange(tile_m.block_size)
|
||||
m_blk = m_idx[:, None, None]
|
||||
for tile_gn, tile_n in hl.tile(
|
||||
[groups_per_row, group_size], block_size=[None, group_size]
|
||||
):
|
||||
gn_idx = tile_gn.index
|
||||
n_offset = tile_n.index
|
||||
n_idx = gn_idx[:, None] * group_size + n_offset[None, :]
|
||||
n_blk = n_idx[None, :, :]
|
||||
mask = (gn_idx < groups_per_row)[None, :, None]
|
||||
|
||||
x_blk = hl.load(input, [m_blk, n_blk], extra_mask=mask).to(
|
||||
dtype=torch.float32
|
||||
)
|
||||
if residual is not None:
|
||||
r_blk = hl.load(residual, [m_blk, n_blk], extra_mask=mask)
|
||||
x_blk = x_blk + r_blk
|
||||
|
||||
w_blk = hl.load(weight, [n_blk], extra_mask=mask)
|
||||
x_norm_blk = (x_blk * rms[:, None, None]).to(input.dtype) * w_blk
|
||||
s_blk = torch.amax(torch.abs(x_norm_blk), dim=-1).to(torch.float32)
|
||||
|
||||
if scale_ub is not None:
|
||||
scale_ub_s = hl.load(scale_ub, [])
|
||||
s_blk = s_blk.clamp(max=scale_ub_s)
|
||||
|
||||
s_blk = s_blk * (1.0 / qtype_max)
|
||||
s_blk = s_blk.clamp(min=min_scaling_factor)
|
||||
|
||||
scale[tile_m, tile_gn] = s_blk
|
||||
|
||||
if quant_dtype == torch.int8:
|
||||
y_blk = (x_norm_blk * (1.0 / s_blk[:, :, None])).round()
|
||||
else:
|
||||
y_blk = x_norm_blk / s_blk[:, :, None]
|
||||
|
||||
y_blk = y_blk.clamp(qtype_traits_min, qtype_traits_max).to(result.dtype)
|
||||
hl.store(result, [m_blk, n_blk], y_blk, extra_mask=mask)
|
||||
|
||||
if residual is not None:
|
||||
hl.store(
|
||||
residual, [m_blk, n_blk], x_blk.to(residual.dtype), extra_mask=mask
|
||||
)
|
||||
@@ -1,273 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from itertools import product
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.utils import (
|
||||
get_fp8_dtype,
|
||||
get_int8_min_max,
|
||||
get_int8_min_scaling_factor,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
raise ImportError(
|
||||
"Helion kernel requires helion to be installed. "
|
||||
"Install it with: pip install helion"
|
||||
)
|
||||
|
||||
import helion
|
||||
import helion.language as hl
|
||||
|
||||
from vllm.kernels.helion.register import register_kernel
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
|
||||
# TODO(xiaohongchen1991): it is difficult for kernel author to cover all input
|
||||
# property combination. Currently, dtypes are fixed. We need optimization to
|
||||
# bucket/skip some combinations
|
||||
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
|
||||
intermediate_size_list = [6144, 12288, 25600]
|
||||
|
||||
in_dtype: torch.dtype = torch.bfloat16
|
||||
out_dtype: torch.dtype = current_platform.fp8_dtype()
|
||||
scale_dtype: torch.dtype = torch.float32
|
||||
group_size_list = [128]
|
||||
inputs = {}
|
||||
for intermediate_size, group_size, num_tokens in product(
|
||||
intermediate_size_list, group_size_list, num_tokens_list
|
||||
):
|
||||
input = torch.randn(
|
||||
num_tokens, 2 * intermediate_size, device="cuda", dtype=in_dtype
|
||||
)
|
||||
result = torch.empty(
|
||||
num_tokens, intermediate_size, device=input.device, dtype=out_dtype
|
||||
)
|
||||
scale = torch.empty(
|
||||
(num_tokens, intermediate_size // group_size),
|
||||
device=input.device,
|
||||
dtype=scale_dtype,
|
||||
)
|
||||
# scale_ub clamps the per-group amax of the SiLU-and-mul activation. Use
|
||||
# a non-degenerate upper bound (midway between the mean and max of the
|
||||
# activation magnitude) so clamping is partially active and the baseline
|
||||
# comparison is meaningful. torch.mean(input) ~= 0 for the zero-mean
|
||||
# input would collapse every scale to the floor and saturate the output.
|
||||
# Mirrors tests/kernels/helion/test_silu_and_mul_per_block_quant.py.
|
||||
act_abs = SiluAndMul.forward_native(input.to(torch.float32)).abs()
|
||||
scale_ub = (0.5 * (act_abs.mean() + act_abs.amax())).to(scale_dtype)
|
||||
|
||||
config_key = CaseKey(
|
||||
{
|
||||
"intermediate_size": intermediate_size,
|
||||
"group_size": group_size,
|
||||
"num_tokens": num_tokens,
|
||||
}
|
||||
)
|
||||
inputs[config_key] = (result, input, scale, group_size, scale_ub, False)
|
||||
|
||||
return inputs
|
||||
|
||||
|
||||
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
|
||||
|
||||
|
||||
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
|
||||
"""Pick the best pre-tuned config for the given input shape.
|
||||
|
||||
Selection strategy:
|
||||
1. Find the closest intermediate_size among available configs
|
||||
(exact match preferred).
|
||||
2. Find the closest group_size among available configs
|
||||
(exact match preferred).
|
||||
3. Among the num_tokens values tuned for that intermediate_size and group_size,
|
||||
pick the smallest num_tokens >= the input's num_tokens. If the input is
|
||||
larger than all available num_tokens, fall back to the largest.
|
||||
"""
|
||||
if not config_keys:
|
||||
return None
|
||||
|
||||
result, _, _, group_size, *_ = args
|
||||
num_tokens, intermediate_size = result.shape
|
||||
|
||||
cache_key = (num_tokens, group_size, intermediate_size)
|
||||
cached = _pick_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
configs: dict[int, dict[int, list[int]]] = {}
|
||||
for key in config_keys:
|
||||
if key.is_default():
|
||||
continue
|
||||
configs.setdefault(key["intermediate_size"], {}).setdefault(
|
||||
key["group_size"], []
|
||||
).append(key["num_tokens"])
|
||||
|
||||
if not configs:
|
||||
return None
|
||||
|
||||
best_intermediate_size = min(configs, key=lambda s: abs(s - intermediate_size))
|
||||
best_group_size = min(
|
||||
configs[best_intermediate_size], key=lambda s: abs(s - group_size)
|
||||
)
|
||||
available_num_tokens = sorted(configs[best_intermediate_size][best_group_size])
|
||||
best_num_tokens = next(
|
||||
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
|
||||
)
|
||||
|
||||
result = CaseKey(
|
||||
{
|
||||
"intermediate_size": best_intermediate_size,
|
||||
"group_size": best_group_size,
|
||||
"num_tokens": best_num_tokens,
|
||||
}
|
||||
)
|
||||
_pick_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
def baseline(
|
||||
out: torch.Tensor, # [num_tokens, intermediate_size]
|
||||
input: torch.Tensor, # [num_tokens, 2 * intermediate_size]
|
||||
scales: torch.Tensor, # [num_tokens, groups_per_row]
|
||||
group_size: int,
|
||||
scale_ub: torch.Tensor | None = None, # scalar tensor
|
||||
is_scale_transposed: bool = False,
|
||||
) -> None:
|
||||
num_tokens, intermediate_size = out.shape
|
||||
groups_per_row = intermediate_size // group_size
|
||||
quant_dtype = out.dtype
|
||||
qtype_min: int | float
|
||||
qtype_max: int | float
|
||||
|
||||
if quant_dtype == torch.int8:
|
||||
qtype_min, qtype_max = get_int8_min_max()
|
||||
min_scaling_factor = get_int8_min_scaling_factor()
|
||||
else:
|
||||
qtype_min, qtype_max = get_fp8_min_max()
|
||||
min_scaling_factor = 1.0 / (qtype_max * 512.0)
|
||||
|
||||
act = SiluAndMul.forward_native(input.to(torch.float32))
|
||||
x_grouped = act.view(num_tokens, groups_per_row, group_size)
|
||||
|
||||
s = torch.amax(torch.abs(x_grouped), dim=-1)
|
||||
if scale_ub is not None:
|
||||
s = s.clamp(max=scale_ub)
|
||||
s = (s * (1.0 / qtype_max)).clamp(min=min_scaling_factor)
|
||||
|
||||
y = x_grouped / s[:, :, None]
|
||||
if quant_dtype == torch.int8:
|
||||
y = y.round()
|
||||
|
||||
scales.copy_(s)
|
||||
out.copy_(
|
||||
y.clamp(qtype_min, qtype_max).view(num_tokens, intermediate_size).to(out.dtype)
|
||||
)
|
||||
|
||||
|
||||
@register_kernel(
|
||||
mutates_args=["out", "scales"],
|
||||
config_picker=pick_config,
|
||||
input_generator=generate_inputs,
|
||||
helion_settings=helion.Settings(
|
||||
autotune_baseline_fn=baseline,
|
||||
ignore_warnings=[helion.exc.TensorOperationInWrapper],
|
||||
),
|
||||
) # type: ignore[misc]
|
||||
def silu_and_mul_per_block_quant(
|
||||
out: torch.Tensor, # [num_tokens, intermediate_size]
|
||||
input: torch.Tensor, # [num_tokens, 2 * intermediate_size]
|
||||
scales: torch.Tensor, # [num_tokens, groups_per_row]
|
||||
group_size: int,
|
||||
scale_ub: torch.Tensor | None = None, # scalar tensor
|
||||
is_scale_transposed: bool = False, # dummy
|
||||
) -> None:
|
||||
# This code assumes batch_dim and num_tokens are flattened
|
||||
assert input.ndim == 2
|
||||
num_tokens, two_intermediate_size = input.shape
|
||||
hl.specialize(two_intermediate_size)
|
||||
|
||||
assert two_intermediate_size % 2 == 0
|
||||
intermediate_size = two_intermediate_size // 2
|
||||
|
||||
assert out.shape[0] == num_tokens
|
||||
assert out.shape[1] == intermediate_size
|
||||
fp8_dtype = get_fp8_dtype()
|
||||
assert out.dtype in [fp8_dtype, torch.int8]
|
||||
|
||||
if scale_ub is not None:
|
||||
assert out.dtype == fp8_dtype
|
||||
assert scale_ub.dtype == torch.float32
|
||||
|
||||
assert scales.ndim == 2 and scales.dtype == torch.float32
|
||||
|
||||
assert scales.shape[0] == num_tokens
|
||||
groups_per_row = scales.shape[1]
|
||||
hl.specialize(groups_per_row)
|
||||
assert (
|
||||
intermediate_size % group_size == 0
|
||||
and intermediate_size // group_size == groups_per_row
|
||||
)
|
||||
|
||||
assert group_size in [64, 128]
|
||||
hl.specialize(group_size)
|
||||
|
||||
assert input.stride()[-1] == 1
|
||||
assert out.stride()[-1] == 1
|
||||
|
||||
quant_dtype = out.dtype
|
||||
qtype_traits_min: int | float
|
||||
qtype_traits_max: int | float
|
||||
if quant_dtype == torch.int8:
|
||||
qtype_traits_min, qtype_traits_max = get_int8_min_max()
|
||||
min_scaling_factor = get_int8_min_scaling_factor()
|
||||
else:
|
||||
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
|
||||
min_scaling_factor = 1.0 / (qtype_traits_max * 512.0)
|
||||
|
||||
qtype_max = float(qtype_traits_max)
|
||||
|
||||
input = input.view(num_tokens, -1, group_size)
|
||||
out = out.view(num_tokens, -1, group_size)
|
||||
|
||||
for tile_m, tile_gn, tile_n in hl.tile(
|
||||
[num_tokens, groups_per_row, group_size], block_size=[1, None, group_size]
|
||||
):
|
||||
x_a_blk = input[tile_m, tile_gn, tile_n].to(torch.float32)
|
||||
x_b_blk = hl.load(
|
||||
input,
|
||||
[tile_m, tile_gn.index + groups_per_row, tile_n],
|
||||
extra_mask=(tile_gn.index + groups_per_row < 2 * groups_per_row)[
|
||||
None, :, None
|
||||
],
|
||||
).to(torch.float32)
|
||||
x_blk = x_a_blk * torch.sigmoid(x_a_blk) * x_b_blk
|
||||
s_blk = torch.amax(torch.abs(x_blk), dim=-1).to(torch.float32)
|
||||
|
||||
if scale_ub is not None:
|
||||
scale_ub_s = hl.load(scale_ub, [])
|
||||
s_blk = s_blk.clamp(max=scale_ub_s)
|
||||
s_blk = s_blk * (1.0 / qtype_max)
|
||||
s_blk = s_blk.clamp(min=min_scaling_factor)
|
||||
|
||||
scales[tile_m, tile_gn] = s_blk
|
||||
if quant_dtype == torch.int8:
|
||||
y_blk = (x_blk * (1.0 / s_blk[:, :, None])).round()
|
||||
else:
|
||||
y_blk = x_blk / s_blk[:, :, None]
|
||||
|
||||
out[tile_m, tile_gn, tile_n] = y_blk.clamp(
|
||||
qtype_traits_min, qtype_traits_max
|
||||
).to(out.dtype)
|
||||
@@ -1,135 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
raise ImportError(
|
||||
"silu_mul_fp8 Helion kernel requires helion to be installed. "
|
||||
"Install it with: pip install helion"
|
||||
)
|
||||
|
||||
import helion.language as hl
|
||||
|
||||
from vllm.kernels.helion.register import register_kernel
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def generate_silu_mul_fp8_inputs() -> dict[CaseKey, tuple[Any, ...]]:
|
||||
intermediate_sizes = [2048, 2880, 4096, 8192, 11008, 14336]
|
||||
|
||||
# Use the same num_tokens values as vLLM's default cudagraph capture sizes.
|
||||
# See vllm/config/vllm.py _set_cudagraph_sizes() for the canonical formula.
|
||||
num_tokens_list = [1, 2, 4] + list(range(8, 256, 8)) + list(range(256, 513, 16))
|
||||
|
||||
inputs: dict[CaseKey, tuple[Any, ...]] = {}
|
||||
for num_tokens in num_tokens_list:
|
||||
for intermediate_size in intermediate_sizes:
|
||||
input_tensor = torch.randn(
|
||||
num_tokens,
|
||||
2 * intermediate_size,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
scale = torch.tensor([1.0], device="cuda", dtype=torch.float32)
|
||||
|
||||
key = CaseKey({"intermediate": intermediate_size, "numtokens": num_tokens})
|
||||
inputs[key] = (input_tensor, scale)
|
||||
|
||||
return inputs
|
||||
|
||||
|
||||
_pick_cache: dict[tuple[int, int], CaseKey | None] = {}
|
||||
|
||||
|
||||
def pick_silu_mul_fp8_config(
|
||||
args: tuple[Any, ...], config_keys: list[CaseKey]
|
||||
) -> CaseKey | None:
|
||||
"""Pick the best pre-tuned config for the given input shape.
|
||||
|
||||
Selection strategy:
|
||||
1. Find the closest intermediate_size among available configs
|
||||
(exact match preferred).
|
||||
2. Among the num_tokens values tuned for that intermediate_size, pick
|
||||
the smallest num_tokens >= the input's num_tokens. If the input is
|
||||
larger than all available num_tokens, fall back to the largest.
|
||||
"""
|
||||
if not config_keys:
|
||||
return None
|
||||
|
||||
input_tensor, _scale = args
|
||||
intermediate_size = int(input_tensor.shape[-1]) // 2
|
||||
num_tokens = int(input_tensor.view(-1, input_tensor.shape[-1]).shape[0])
|
||||
|
||||
cache_key = (num_tokens, intermediate_size)
|
||||
cached = _pick_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
by_isize: dict[int, list[int]] = {}
|
||||
for k in config_keys:
|
||||
if k.is_default():
|
||||
continue
|
||||
by_isize.setdefault(k["intermediate"], []).append(k["numtokens"])
|
||||
|
||||
if not by_isize:
|
||||
return None
|
||||
|
||||
best_isize = min(by_isize, key=lambda s: abs(s - intermediate_size))
|
||||
available = sorted(by_isize[best_isize])
|
||||
best_ntokens = next((n for n in available if n >= num_tokens), available[-1])
|
||||
|
||||
result = CaseKey({"intermediate": best_isize, "numtokens": best_ntokens})
|
||||
_pick_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
@register_kernel(
|
||||
config_picker=pick_silu_mul_fp8_config,
|
||||
input_generator=generate_silu_mul_fp8_inputs,
|
||||
)
|
||||
def silu_mul_fp8(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
original_shape = input.shape
|
||||
two_d = hl.specialize(original_shape[-1])
|
||||
d = two_d // 2
|
||||
output_shape = original_shape[:-1] + (d,)
|
||||
|
||||
input_2d = input.view(-1, original_shape[-1])
|
||||
m = input_2d.shape[0]
|
||||
|
||||
# TODO(gmagogsfm): Support for more float8 subtypes (e4m3fnuz, e5m2) coming
|
||||
out = torch.empty((m, d), device=input.device, dtype=torch.float8_e4m3fn)
|
||||
|
||||
input_part_a = input_2d[:, :d]
|
||||
input_part_b = input_2d[:, d:]
|
||||
|
||||
assert scale.numel() == 1, "Scale must be a scalar Tensor"
|
||||
|
||||
for tile_m, tile_n in hl.tile([m, d]):
|
||||
a_vals = input_part_a[tile_m, tile_n]
|
||||
silu_result = torch.nn.functional.silu(a_vals)
|
||||
b_vals = input_part_b[tile_m, tile_n]
|
||||
result = silu_result * b_vals
|
||||
result_f32 = result.to(torch.float32)
|
||||
scale_val = hl.load(scale, [0])
|
||||
inv_scale = 1.0 / scale_val
|
||||
result_scaled = result_f32 * inv_scale
|
||||
out[tile_m, tile_n] = result_scaled.to(out.dtype)
|
||||
|
||||
return out.view(output_shape)
|
||||
|
||||
|
||||
def silu_mul_fp8_baseline(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
output_shape = input.shape[:-1] + (input.shape[-1] // 2,)
|
||||
out = torch.empty(output_shape, dtype=torch.float8_e4m3fn, device=input.device)
|
||||
torch.ops._C.silu_and_mul_quant(out, input, scale)
|
||||
return out
|
||||
@@ -1,513 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM Helion kernel registration with pre-tuned config selection.
|
||||
|
||||
This module leverages Helion's internal config selection infrastructure to use
|
||||
pre-tuned configs instead of runtime autotuning.
|
||||
|
||||
How Helion Normally Works
|
||||
-------------------------
|
||||
For each kernel invocation, Helion:
|
||||
1. Computes a cache key from input arguments
|
||||
2. Looks up the key in its internal compilation cache
|
||||
3. On cache miss, runs autotuning to find the best config
|
||||
4. Compiles and caches the kernel with that config
|
||||
|
||||
How We Override It
|
||||
------------------
|
||||
We override two Helion hooks to use pre-tuned configs:
|
||||
|
||||
1. **key**: We provide a key function (derived from config_picker) that
|
||||
computes cache keys matching our pre-tuned config keys. This ensures Helion's
|
||||
internal cache uses keys that correspond to configs we've prepared.
|
||||
|
||||
2. **autotuner_fn**: We provide PresetConfigSearch which, instead of autotuning,
|
||||
simply returns the pre-tuned config for the computed key. On cache miss,
|
||||
Helion calls our autotuner which returns the author-prepared config.
|
||||
|
||||
Both hooks use the same config_picker logic to ensure the cache key computed
|
||||
by key matches the config returned by the autotuner.
|
||||
|
||||
Key Classes
|
||||
-----------
|
||||
- HelionKernelWrapper: Wraps raw kernel + config_picker, creates configured kernels
|
||||
- ConfiguredHelionKernel: Platform-specific kernel with pre-tuned configs
|
||||
- PresetConfigSearch: Custom autotuner that returns pre-tuned configs
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch.library import Library
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import has_helion
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
if not has_helion():
|
||||
raise ImportError(
|
||||
"register module requires helion to be installed. "
|
||||
"Install it with: pip install helion"
|
||||
)
|
||||
|
||||
import helion
|
||||
from helion.autotuner.base_search import BaseAutotuner
|
||||
from helion.runtime.config import Config
|
||||
from helion.runtime.settings import default_autotuner_fn
|
||||
|
||||
# TODO(gmagogsfm): Remove CustomOp fallback path (_get_or_register_custom_op,
|
||||
# vllm_helion_lib, direct_register_custom_op) once vLLM requires PyTorch >= 2.11.
|
||||
# FIXME(gmagogsfm): Re-enable HOP path once performance regression is fixed.
|
||||
# _HOP_AVAILABLE = requires_torch_version("2.11")
|
||||
_HOP_AVAILABLE = False
|
||||
|
||||
if _HOP_AVAILABLE:
|
||||
from helion._compat import supports_torch_compile_fusion
|
||||
from helion._compiler._dynamo.higher_order_ops import helion_kernel_side_table
|
||||
from helion._compiler._dynamo.variables import HelionKernelVariable
|
||||
from helion.runtime.kernel import Kernel
|
||||
from torch._dynamo.guards import GuardBuilder
|
||||
from torch._dynamo.variables.builder import VariableBuilder
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
vllm_helion_lib = Library("vllm_helion", "FRAGMENT") # noqa
|
||||
|
||||
ConfigPicker = Callable[[tuple[Any, ...], list[CaseKey]], CaseKey | None]
|
||||
|
||||
|
||||
def validate_helion_settings(
|
||||
helion_settings: helion.Settings | None, op_name: str
|
||||
) -> None:
|
||||
if helion_settings is None:
|
||||
return
|
||||
|
||||
settings_dict = helion_settings.to_dict()
|
||||
|
||||
if (
|
||||
"autotuner_fn" in settings_dict
|
||||
and settings_dict["autotuner_fn"] is not None
|
||||
and settings_dict["autotuner_fn"] is not default_autotuner_fn
|
||||
):
|
||||
raise ValueError(
|
||||
f"HelionKernelWrapper for '{op_name}' uses a custom autotuner via "
|
||||
f"config picker. Remove 'autotuner_fn' from helion_settings and use "
|
||||
f"register_kernel(..., config_picker=...) instead."
|
||||
)
|
||||
|
||||
if settings_dict.get("static_shapes") is True:
|
||||
logger.warning(
|
||||
"Kernel '%s' has static_shapes=True in helion_settings, "
|
||||
"which will be overridden to False. vLLM requires dynamic "
|
||||
"shapes for variable batch sizes and sequence lengths.",
|
||||
op_name,
|
||||
)
|
||||
|
||||
|
||||
def create_helion_decorated_kernel(
|
||||
raw_kernel_func: Callable,
|
||||
helion_settings: helion.Settings | None = None,
|
||||
extra_kwargs: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
kernel_kwargs: dict[str, Any] = {}
|
||||
if helion_settings:
|
||||
kernel_kwargs.update(helion_settings.to_dict())
|
||||
|
||||
# vLLM requires dynamic shapes for variable batch sizes and sequence lengths
|
||||
kernel_kwargs["static_shapes"] = False
|
||||
|
||||
if extra_kwargs:
|
||||
kernel_kwargs.update(extra_kwargs)
|
||||
|
||||
return helion.kernel(**kernel_kwargs)(raw_kernel_func)
|
||||
|
||||
|
||||
class PresetConfigSearch(BaseAutotuner):
|
||||
"""Custom autotuner that uses a preset config selector instead of autotuning."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
args: tuple[Any, ...],
|
||||
config_selector: Callable[[tuple[Any, ...]], Config],
|
||||
):
|
||||
self.args = args
|
||||
self.config_selector = config_selector
|
||||
|
||||
def autotune(self, *, skip_cache: bool = False) -> Config:
|
||||
return self.config_selector(self.args)
|
||||
|
||||
|
||||
class ConfiguredHelionKernel:
|
||||
"""A configured Helion kernel bound to a specific platform."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
op_name: str,
|
||||
config_picker: ConfigPicker | None,
|
||||
raw_kernel_func: Callable,
|
||||
helion_settings: helion.Settings | None = None,
|
||||
):
|
||||
self.op_name = op_name
|
||||
self.config_picker = config_picker
|
||||
self.raw_kernel_func = raw_kernel_func
|
||||
self.helion_settings = helion_settings
|
||||
self._decorated_kernel = self._create_decorated_kernel()
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return self._decorated_kernel(*args, **kwargs)
|
||||
|
||||
def _create_key_computer(self):
|
||||
"""Create a key computer function derived from the config picker.
|
||||
|
||||
The returned function receives kernel arguments unpacked (*args) to match
|
||||
Helion's key signature (called as self._key_fn(*args)).
|
||||
"""
|
||||
if self.config_picker is None:
|
||||
raise RuntimeError(
|
||||
f"No config picker registered for kernel '{self.op_name}'. "
|
||||
f"A config_picker must be provided to register_kernel()."
|
||||
)
|
||||
|
||||
picker = self.config_picker
|
||||
all_keys = list(self.configs.keys())
|
||||
default = CaseKey.default()
|
||||
has_default = default in self.configs
|
||||
|
||||
def key_computer(*args):
|
||||
selected = picker(args, all_keys)
|
||||
if selected is not None:
|
||||
return str(selected)
|
||||
if has_default:
|
||||
return str(default)
|
||||
return None
|
||||
|
||||
return key_computer
|
||||
|
||||
def _create_config_selector(self, key_computer):
|
||||
str_to_key = {str(k): k for k in self.configs}
|
||||
|
||||
def config_selector(args):
|
||||
selected_str = key_computer(*args)
|
||||
|
||||
if selected_str is None:
|
||||
raise ValueError(
|
||||
f"Config picker returned None for kernel "
|
||||
f"'{self.op_name}' with available config keys: "
|
||||
f"{list(self.configs.keys())}"
|
||||
)
|
||||
|
||||
config_key = str_to_key.get(selected_str)
|
||||
if config_key is None:
|
||||
raise ValueError(
|
||||
f"Config picker returned invalid config key "
|
||||
f"'{selected_str}' for kernel "
|
||||
f"'{self.op_name}'. "
|
||||
f"Available keys: {list(self.configs.keys())}"
|
||||
)
|
||||
|
||||
return self.configs[config_key]
|
||||
|
||||
return config_selector
|
||||
|
||||
def _load_platform_configs(self) -> None:
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.utils import get_canonical_gpu_name
|
||||
|
||||
self.platform = get_canonical_gpu_name()
|
||||
config_manager = ConfigManager()
|
||||
self.configs = config_manager.get_platform_configs(self.op_name, self.platform)
|
||||
|
||||
if not self.configs:
|
||||
raise ValueError(
|
||||
f"No configs available for kernel '{self.op_name}' "
|
||||
f"on platform '{self.platform}'"
|
||||
)
|
||||
|
||||
def _create_decorated_kernel(self) -> Callable[..., Any]:
|
||||
self._load_platform_configs()
|
||||
|
||||
key_computer = self._create_key_computer()
|
||||
config_selector = self._create_config_selector(key_computer)
|
||||
|
||||
extra_kwargs = {
|
||||
"autotuner_fn": lambda _, args: PresetConfigSearch(args, config_selector),
|
||||
"key": key_computer,
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
"Creating decorated kernel %s with custom autotuner on platform %s",
|
||||
self.op_name,
|
||||
self.platform,
|
||||
)
|
||||
return create_helion_decorated_kernel(
|
||||
self.raw_kernel_func, self.helion_settings, extra_kwargs
|
||||
)
|
||||
|
||||
|
||||
class HelionKernelWrapper:
|
||||
"""Wrapper for Helion kernels with pre-tuned config selection and HOP support."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_kernel_func: Callable,
|
||||
op_name: str,
|
||||
fake_impl: Callable | None,
|
||||
config_picker: ConfigPicker,
|
||||
mutates_args: list[str] | None = None,
|
||||
helion_settings: helion.Settings | None = None,
|
||||
input_generator: (Callable[[], dict[CaseKey, tuple[Any, ...]]] | None) = None,
|
||||
):
|
||||
# Validate helion_settings doesn't conflict with our custom autotuner
|
||||
validate_helion_settings(helion_settings, op_name)
|
||||
|
||||
self.raw_kernel_func = raw_kernel_func
|
||||
self.op_name = op_name
|
||||
self._fake_impl = fake_impl
|
||||
self.helion_settings = helion_settings
|
||||
self._config_picker = config_picker
|
||||
self._input_generator = input_generator
|
||||
self._mutates_args = mutates_args
|
||||
self._configured_kernel: ConfiguredHelionKernel | None = None
|
||||
# TODO(@gmagogsfm): Remove this disable flag once integrated with vLLM IR,
|
||||
# which handles op enablement/disablement.
|
||||
self._disabled = False
|
||||
self._disabled_reason: str | None = None
|
||||
|
||||
try:
|
||||
if not _HOP_AVAILABLE:
|
||||
self._get_or_register_custom_op()
|
||||
else:
|
||||
self.get_configured_op()
|
||||
except ValueError as e:
|
||||
self._disabled = True
|
||||
self._disabled_reason = str(e)
|
||||
logger.warning(
|
||||
"Helion kernel '%s' is disabled: %s",
|
||||
op_name,
|
||||
self._disabled_reason,
|
||||
)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
if self._disabled:
|
||||
raise RuntimeError(
|
||||
f"Helion kernel '{self.op_name}' is disabled: {self._disabled_reason}"
|
||||
)
|
||||
if not _HOP_AVAILABLE:
|
||||
op = getattr(torch.ops.vllm_helion, self.op_name)
|
||||
return op(*args, **kwargs)
|
||||
assert self._configured_kernel is not None, (
|
||||
f"Kernel '{self.op_name}' was not initialized. "
|
||||
"Please open an issue on GitHub."
|
||||
)
|
||||
|
||||
# During Dynamo tracing, this call will be intercepted by our custom
|
||||
# HelionKernelWrapperVariable and handled via proper HOP emission.
|
||||
# During eager execution, call the kernel directly.
|
||||
return self._configured_kernel(*args, **kwargs)
|
||||
|
||||
def get_inputs(self) -> dict[CaseKey, tuple[Any, ...]]:
|
||||
if self._input_generator is None:
|
||||
raise NotImplementedError(
|
||||
f"No input generator registered for kernel '{self.op_name}'. "
|
||||
f"Use register_kernel(..., input_generator=...) to register one."
|
||||
)
|
||||
return self._input_generator()
|
||||
|
||||
def run_autotune(
|
||||
self,
|
||||
inputs: tuple[Any, ...],
|
||||
autotune_effort: str = "quick",
|
||||
) -> Config:
|
||||
"""Run autotuning for a single input configuration."""
|
||||
extra_kwargs = {
|
||||
"autotune_effort": autotune_effort,
|
||||
"autotune_ignore_errors": True,
|
||||
}
|
||||
autotune_kernel = create_helion_decorated_kernel(
|
||||
self.raw_kernel_func, self.helion_settings, extra_kwargs
|
||||
)
|
||||
return autotune_kernel.autotune(inputs)
|
||||
|
||||
def get_configured_op(self) -> ConfiguredHelionKernel:
|
||||
if self._disabled:
|
||||
raise RuntimeError(
|
||||
f"Helion kernel '{self.op_name}' is disabled: {self._disabled_reason}"
|
||||
)
|
||||
if self._configured_kernel is None:
|
||||
self._configured_kernel = ConfiguredHelionKernel(
|
||||
op_name=self.op_name,
|
||||
config_picker=self._config_picker,
|
||||
raw_kernel_func=self.raw_kernel_func,
|
||||
helion_settings=self.helion_settings,
|
||||
)
|
||||
return self._configured_kernel
|
||||
|
||||
def _get_or_register_custom_op(self) -> Any:
|
||||
if hasattr(torch.ops.vllm_helion, self.op_name):
|
||||
return getattr(torch.ops.vllm_helion, self.op_name)
|
||||
|
||||
configured_kernel = self.get_configured_op()
|
||||
|
||||
logger.info("Registering op: vllm_helion::%s", self.op_name)
|
||||
direct_register_custom_op(
|
||||
op_name=self.op_name,
|
||||
op_func=configured_kernel._decorated_kernel,
|
||||
mutates_args=self._mutates_args,
|
||||
fake_impl=self._fake_impl,
|
||||
target_lib=vllm_helion_lib,
|
||||
)
|
||||
return getattr(torch.ops.vllm_helion, self.op_name)
|
||||
|
||||
|
||||
# Global registry for tracking all registered HelionKernelWrapper instances
|
||||
_REGISTERED_KERNELS: dict[str, HelionKernelWrapper] = {}
|
||||
|
||||
|
||||
def get_registered_kernels() -> dict[str, HelionKernelWrapper]:
|
||||
return _REGISTERED_KERNELS.copy()
|
||||
|
||||
|
||||
def get_kernel_by_name(kernel_name: str) -> HelionKernelWrapper | None:
|
||||
return _REGISTERED_KERNELS.get(kernel_name)
|
||||
|
||||
|
||||
def infer_fake_impl(
|
||||
kernel_func: Callable,
|
||||
helion_settings: helion.Settings | None = None,
|
||||
) -> Callable:
|
||||
def helion_fake_kernel(*args, **kwargs):
|
||||
kernel_kwargs = {}
|
||||
if helion_settings:
|
||||
kernel_kwargs.update(helion_settings.to_dict())
|
||||
|
||||
temp_decorated_kernel = helion.kernel(**kernel_kwargs)(kernel_func)
|
||||
|
||||
# Bind with args to get config_spec, then get a valid default config
|
||||
bound = temp_decorated_kernel.bind(args)
|
||||
default_config = bound.config_spec.default_config()
|
||||
compiled_runner = bound.compile_config(default_config)
|
||||
|
||||
return compiled_runner(*args, **kwargs, _launcher=lambda *a, **kw: None)
|
||||
|
||||
return helion_fake_kernel
|
||||
|
||||
|
||||
def register_kernel(
|
||||
op_name: str | None = None,
|
||||
*,
|
||||
config_picker: ConfigPicker,
|
||||
fake_impl: Callable | None = None,
|
||||
mutates_args: list[str] | None = None,
|
||||
helion_settings: helion.Settings | None = None,
|
||||
input_generator: (Callable[[], dict[CaseKey, tuple[Any, ...]]] | None) = None,
|
||||
) -> Callable[[Callable], HelionKernelWrapper]:
|
||||
"""Register a Helion kernel with pre-tuned config selection.
|
||||
|
||||
Args:
|
||||
op_name: Name to register the op under. Defaults to the function name.
|
||||
fake_impl: Optional meta/fake implementation for tracing.
|
||||
mutates_args: Names of arguments the kernel mutates in place.
|
||||
helion_settings: Optional Helion settings override.
|
||||
config_picker: Required. Receives ``(args, config_keys)``
|
||||
where each config key is a ``dict[str, Any]`` mapping
|
||||
parameter names to values. Return the best-matching
|
||||
dict, or ``None`` to fall back to the default config.
|
||||
|
||||
Example::
|
||||
|
||||
def pick_config(args, config_keys):
|
||||
x = args[0]
|
||||
best = min(config_keys, key=lambda k: abs(k["size"] - x.shape[0]))
|
||||
return best
|
||||
|
||||
input_generator: Optional. Returns ``dict[str, tuple]`` where
|
||||
each key is a serialized config key and each value is a
|
||||
tuple of arguments to pass to the kernel.
|
||||
|
||||
Example::
|
||||
|
||||
def generate_inputs():
|
||||
return {
|
||||
"4096": (torch.randn(4096, device="cuda"), 0.5),
|
||||
"8192": (torch.randn(8192, device="cuda"), 0.5),
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
def decorator(kernel_func: Callable) -> HelionKernelWrapper:
|
||||
final_op_name = op_name if op_name else kernel_func.__name__
|
||||
|
||||
if final_op_name in _REGISTERED_KERNELS:
|
||||
raise ValueError(
|
||||
f"Helion kernel '{final_op_name}' is already registered. "
|
||||
f"Use a different op_name or check for duplicate registrations."
|
||||
)
|
||||
|
||||
# A void-returning kernel that mutates its args needs no fake impl:
|
||||
# PyTorch auto-generates a trivial one for mutable, no-return schemas.
|
||||
returns_none = inspect.signature(kernel_func).return_annotation is None
|
||||
|
||||
final_fake_impl = fake_impl
|
||||
if final_fake_impl is None and not (mutates_args and returns_none):
|
||||
final_fake_impl = infer_fake_impl(kernel_func, helion_settings)
|
||||
logger.debug(
|
||||
"Auto-generated fake_impl for Helion kernel '%s'",
|
||||
kernel_func.__name__,
|
||||
)
|
||||
|
||||
kernel_wrapper = HelionKernelWrapper(
|
||||
raw_kernel_func=kernel_func,
|
||||
op_name=final_op_name,
|
||||
fake_impl=final_fake_impl,
|
||||
config_picker=config_picker,
|
||||
mutates_args=mutates_args,
|
||||
helion_settings=helion_settings,
|
||||
input_generator=input_generator,
|
||||
)
|
||||
|
||||
_REGISTERED_KERNELS[final_op_name] = kernel_wrapper
|
||||
|
||||
logger.info(
|
||||
"Registered Helion kernel '%s' as HelionKernelWrapper",
|
||||
kernel_func.__name__,
|
||||
)
|
||||
|
||||
return kernel_wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
# Register HelionKernelWrapper with Dynamo's variable tracker system
|
||||
if _HOP_AVAILABLE:
|
||||
|
||||
def _register_vllm_helion_dynamo_variable():
|
||||
"""Register HelionKernelWrapper with Dynamo's VariableBuilder.
|
||||
|
||||
When Dynamo encounters a HelionKernelWrapper during tracing, this
|
||||
extracts the underlying Helion Kernel and delegates to Helion's own
|
||||
registered Kernel handler, which handles HOP emission, side table
|
||||
registration, and inductor lowering setup.
|
||||
"""
|
||||
|
||||
def wrap_helion_kernel_wrapper(
|
||||
builder: VariableBuilder, value: HelionKernelWrapper
|
||||
):
|
||||
kernel = value.get_configured_op()._decorated_kernel
|
||||
if supports_torch_compile_fusion():
|
||||
helion_handler = VariableBuilder._type_dispatch()[Kernel]
|
||||
return helion_handler(builder, kernel)
|
||||
kernel_idx = helion_kernel_side_table.add_kernel(kernel)
|
||||
builder.install_guards(GuardBuilder.ID_MATCH)
|
||||
return HelionKernelVariable(kernel, kernel_idx, source=builder.source)
|
||||
|
||||
dispatch = VariableBuilder._type_dispatch()
|
||||
dispatch[HelionKernelWrapper] = wrap_helion_kernel_wrapper
|
||||
|
||||
# Register immediately when the module is imported
|
||||
_register_vllm_helion_dynamo_variable()
|
||||
@@ -1,93 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility functions for Helion kernel management."""
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Maps known variant GPU names (after lowercase/underscore normalization)
|
||||
# to their canonical form.
|
||||
#
|
||||
# Names that are already canonical after normalization are NOT listed here.
|
||||
# For example, "NVIDIA H200" normalizes to "nvidia_h200" which needs no
|
||||
# further mapping, and AMD ROCm names like "AMD_Instinct_MI300X" come from
|
||||
# a controlled lookup table in rocm.py and normalize cleanly to
|
||||
# "amd_instinct_mi300x". Only names with variant suffixes (form factor,
|
||||
# memory size, memory type, etc.) that should be stripped need entries.
|
||||
#
|
||||
# To add a new GPU variant: run `canonicalize_gpu_name()` without the alias
|
||||
# to see the normalized name, then add a mapping here if it contains variant
|
||||
# suffixes that should be stripped (e.g. Blackwell/Rubin variants).
|
||||
_GPU_NAME_ALIASES: dict[str, str] = {
|
||||
# H100 variants
|
||||
"nvidia_h100_pcie": "nvidia_h100",
|
||||
"nvidia_h100_sxm5": "nvidia_h100",
|
||||
"nvidia_h100_80gb_hbm3": "nvidia_h100",
|
||||
"nvidia_h100_nvl": "nvidia_h100",
|
||||
# H200 variants
|
||||
"nvidia_h200_nvl": "nvidia_h200",
|
||||
"nvidia_h200_141gb_hbm3e": "nvidia_h200",
|
||||
# A100 variants
|
||||
"nvidia_a100_sxm4_80gb": "nvidia_a100",
|
||||
"nvidia_a100_sxm4_40gb": "nvidia_a100",
|
||||
"nvidia_a100_pcie_80gb": "nvidia_a100",
|
||||
"nvidia_a100_pcie_40gb": "nvidia_a100",
|
||||
"nvidia_a100_80gb_pcie": "nvidia_a100",
|
||||
# V100 variants (Tesla-branded)
|
||||
"tesla_v100_sxm2_32gb": "tesla_v100",
|
||||
"tesla_v100_sxm2_16gb": "tesla_v100",
|
||||
"tesla_v100_pcie_32gb": "tesla_v100",
|
||||
"tesla_v100_pcie_16gb": "tesla_v100",
|
||||
# AMD ROCm variants (from _ROCM_DEVICE_ID_NAME_MAP in rocm.py)
|
||||
"amd_instinct_mi300x_hf": "amd_instinct_mi300x",
|
||||
# ADD MORE HERE
|
||||
}
|
||||
|
||||
|
||||
def get_gpu_name(device_id: int | None = None) -> str:
|
||||
if device_id is None:
|
||||
logger.warning_once(
|
||||
"get_gpu_name() called without device_id, defaulting to 0. "
|
||||
"This may return the wrong device name in multi-node setups."
|
||||
)
|
||||
device_id = 0
|
||||
return current_platform.get_device_name(device_id)
|
||||
|
||||
|
||||
def canonicalize_gpu_name(name: str) -> str:
|
||||
"""Canonicalize GPU name for use as a platform identifier.
|
||||
|
||||
Converts to lowercase, replaces separators with underscores,
|
||||
and maps known variant names to their canonical form via _GPU_NAME_ALIASES.
|
||||
e.g., "NVIDIA H100 80GB HBM3" -> "nvidia_h100"
|
||||
"NVIDIA A100-SXM4-80GB" -> "nvidia_a100"
|
||||
"AMD Instinct MI300X" -> "amd_instinct_mi300x"
|
||||
"""
|
||||
if not name or not name.strip():
|
||||
raise ValueError("GPU name cannot be empty")
|
||||
name = re.sub(r"[\s/-]+", "_", name.lower())
|
||||
if name in _GPU_NAME_ALIASES:
|
||||
return _GPU_NAME_ALIASES[name]
|
||||
return name
|
||||
|
||||
|
||||
def get_canonical_gpu_name(device_id: int | None = None) -> str:
|
||||
return canonicalize_gpu_name(get_gpu_name(device_id))
|
||||
|
||||
|
||||
def get_fp8_dtype() -> torch.dtype:
|
||||
return current_platform.fp8_dtype()
|
||||
|
||||
|
||||
def get_int8_min_max() -> tuple[int, int]:
|
||||
qtype_traits = torch.iinfo(torch.int8)
|
||||
return qtype_traits.min, qtype_traits.max
|
||||
|
||||
|
||||
def get_int8_min_scaling_factor() -> float:
|
||||
return torch.finfo(torch.float32).eps
|
||||
@@ -1,127 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""This file registers Oink implementations for vLLM IR ops.
|
||||
|
||||
vLLM does not depend on the external Oink repository/package. When an external
|
||||
plugin registers torch.library.custom_op entrypoints under the `oink::`
|
||||
namespace (e.g. via vLLM's general_plugins mechanism), these ops will be marked
|
||||
as supported. To dispatch to those ops, set kernel_config.ir_op_priority.<op> to oink.
|
||||
Alternatively, `VLLM_USE_OINK_OPS=1` will add this to priority by default.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from vllm import ir
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
OINK_AVAILABLE = current_platform.has_device_capability(100) and hasattr(
|
||||
torch.ops, "oink"
|
||||
)
|
||||
|
||||
|
||||
def has_oink_op(name: str) -> bool:
|
||||
"""Check if a specific oink op is registered."""
|
||||
return OINK_AVAILABLE and hasattr(torch.ops.oink, name)
|
||||
|
||||
|
||||
def _can_view_as_2d(x: Tensor) -> bool:
|
||||
"""Return True if x.view(-1, x.shape[-1]) is viewable (no copy)."""
|
||||
if x.dim() < 2:
|
||||
return False
|
||||
if x.dim() == 2:
|
||||
return True
|
||||
# For a view(-1, N) to be valid, all leading dims must be contiguous with
|
||||
# respect to each other (size-1 dims are ignored).
|
||||
for dim in range(x.dim() - 1):
|
||||
# Strides for size-1 dims are irrelevant and can be arbitrary.
|
||||
if x.size(dim + 1) != 1 and x.stride(dim) != x.stride(dim + 1) * x.size(
|
||||
dim + 1
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _is_oink_stride_compatible_2d(x_2d: Tensor) -> bool:
|
||||
"""Return True if x_2d meets Oink's pointer-path stride constraints."""
|
||||
if x_2d.dim() != 2:
|
||||
return False
|
||||
if x_2d.stride(1) != 1:
|
||||
return False
|
||||
# Match Oink's vectorization constraint: stride(0) divisible by 256b.
|
||||
if x_2d.dtype in (torch.float16, torch.bfloat16):
|
||||
divby = 16
|
||||
elif x_2d.dtype == torch.float32:
|
||||
divby = 8
|
||||
else:
|
||||
return False
|
||||
return (x_2d.stride(0) % divby) == 0
|
||||
|
||||
|
||||
oink_rms_supported = (
|
||||
lambda x, weight, epsilon, variance_size=None: variance_size is None
|
||||
and weight is not None
|
||||
and x.dim() >= 2
|
||||
and x.dtype == weight.dtype
|
||||
and weight.is_contiguous()
|
||||
and _can_view_as_2d(x)
|
||||
and _is_oink_stride_compatible_2d(x.view(-1, x.shape[-1]))
|
||||
)
|
||||
"""
|
||||
Oink rms only supports 2d-like inputs with contiguous weight
|
||||
and no variance_size override.
|
||||
"""
|
||||
|
||||
|
||||
@ir.ops.rms_norm.register_impl(
|
||||
"oink", supports_args=oink_rms_supported, supported=has_oink_op("rmsnorm")
|
||||
)
|
||||
def rms_norm(
|
||||
x: Tensor,
|
||||
weight: Tensor | None,
|
||||
epsilon: float,
|
||||
variance_size: int | None = None,
|
||||
) -> Tensor:
|
||||
assert variance_size is None
|
||||
x_2d = x.view(-1, x.shape[-1])
|
||||
return torch.ops.oink.rmsnorm(x_2d, weight, epsilon).view_as(x)
|
||||
|
||||
|
||||
oink_add_rms_supported = (
|
||||
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
|
||||
and weight is not None
|
||||
and x.dim() >= 2
|
||||
and x.dtype == weight.dtype
|
||||
and weight.is_contiguous()
|
||||
and _can_view_as_2d(x)
|
||||
and _is_oink_stride_compatible_2d(x.view(-1, x.shape[-1]))
|
||||
# residual must have 2d-compatible strides and match x shape/dtype
|
||||
and x.dtype == x_residual.dtype
|
||||
and x.shape == x_residual.shape
|
||||
and _can_view_as_2d(x_residual)
|
||||
and _is_oink_stride_compatible_2d(x_residual.view(-1, x_residual.shape[-1]))
|
||||
)
|
||||
"""
|
||||
Oink fused_add_rms_norm has the same constraints as rms_norm,
|
||||
and residual must be 2d-like with compatible strides.
|
||||
"""
|
||||
|
||||
|
||||
@ir.ops.fused_add_rms_norm.register_impl(
|
||||
"oink",
|
||||
supports_args=oink_add_rms_supported,
|
||||
supported=has_oink_op("fused_add_rms_norm"),
|
||||
inplace=True,
|
||||
)
|
||||
def fused_add_rms_norm(
|
||||
x: Tensor,
|
||||
x_residual: Tensor,
|
||||
weight: Tensor | None,
|
||||
epsilon: float,
|
||||
variance_size: int | None = None,
|
||||
) -> tuple[Tensor, Tensor]:
|
||||
assert variance_size is None
|
||||
x_2d = x.view(-1, x.shape[-1])
|
||||
residual_2d = x_residual.view(-1, x_residual.shape[-1])
|
||||
torch.ops.oink.fused_add_rms_norm(x_2d, residual_2d, weight, epsilon)
|
||||
return x, x_residual
|
||||
Reference in New Issue
Block a user