[Kernel][Helion] Optimize Helion config parsing latency (#40850)

Signed-off-by: Yanan Cao <gmagogsfm@gmail.com>
Co-authored-by: Claude Sonnet 4 <noreply@anthropic.com>
This commit is contained in:
Yanan Cao
2026-05-07 20:27:34 -07:00
committed by GitHub
co-authored by Claude Sonnet 4
parent baf068d8be
commit 0b99971352
13 changed files with 31670 additions and 27774 deletions
+4
View File
@@ -3,11 +3,13 @@
"""Helion integration for vLLM."""
import vllm.kernels.helion.ops # noqa: F401 Auto-register all Helion ops
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import (
ConfigManager,
ConfigSet,
)
from vllm.kernels.helion.register import (
ConfigPicker,
ConfiguredHelionKernel,
HelionKernelWrapper,
get_kernel_by_name,
@@ -19,9 +21,11 @@ from vllm.kernels.helion.utils import canonicalize_gpu_name, get_canonical_gpu_n
__all__ = [
# Config management
"CaseKey",
"ConfigManager",
"ConfigSet",
# Kernel registration
"ConfigPicker",
"ConfiguredHelionKernel",
"HelionKernelWrapper",
"get_kernel_by_name",
+72
View File
@@ -0,0 +1,72 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Structured key for identifying kernel config/autotune/benchmark cases.
"""
from __future__ import annotations
import json
from typing import Any
class CaseKey(dict[str, Any]):
"""Immutable, hashable dict for identifying kernel cases.
Used as the key for config lookup, autotuning, benchmarking, and
input generation. Behaves like a read-only dict and can be used
as a dict key or in sets.
The canonical string form (``__str__``) is stable JSON with sorted
keys. Use ``CaseKey.default()`` for the default/fallback key.
The regular constructor requires at least one key-value pair::
CaseKey({"intermediate": 2048, "numtokens": 256})
CaseKey.default() # default/fallback
"""
def __init__(self, *args: Any, _allow_empty: bool = False, **kwargs: Any):
super().__init__(*args, **kwargs)
if not self and not _allow_empty:
raise TypeError(
"CaseKey requires at least one key-value pair. "
"Use CaseKey.default() for the default config key."
)
self._str: str | None = None
self._hash: int | None = None
@classmethod
def default(cls) -> CaseKey:
"""Create a default case key (empty)."""
return cls(_allow_empty=True)
def __hash__(self) -> int: # type: ignore[override]
if self._hash is None:
self._hash = hash(str(self))
return self._hash
def __str__(self) -> str:
if self._str is None:
self._str = json.dumps(dict(self), sort_keys=True, separators=(",", ":"))
return self._str
def __repr__(self) -> str:
if not self:
return "CaseKey.default()"
return f"CaseKey({dict(self)})"
def is_default(self) -> bool:
"""Return True if this is the default case key (empty)."""
return not self
def _readonly(self, *args: Any, **kwargs: Any) -> Any:
raise TypeError("CaseKey is immutable")
__setitem__ = _readonly # type: ignore[assignment]
__delitem__ = _readonly # type: ignore[assignment]
__ior__ = _readonly # type: ignore[assignment]
update = _readonly # type: ignore[assignment]
pop = _readonly # type: ignore[assignment]
popitem = _readonly # type: ignore[assignment]
setdefault = _readonly # type: ignore[assignment]
clear = _readonly # type: ignore[assignment]
+90 -56
View File
@@ -11,25 +11,25 @@ Config File Structure
Each kernel has a directory: {kernel_name}/
Inside, each GPU platform has its own JSON file: {kernel_name}/{platform}.json
For example:
silu_mul_fp8/
nvidia_h100.json # { "default": {...}, "batch_32_hidden_4096": {...} }
nvidia_h200.json # { "batch_16_hidden_2048": {...} }
Platform files store config entries as a JSON array::
Each platform file maps config keys to Helion config objects.
Config keys should be structured strings that encode the relevant
parameters (e.g., "batch_32_hidden_4096", "seq_512_heads_16", "fp8_batch_64", etc.).
[
{"key": {}, "config": {...}},
{"key": {"intermediate": 2048, "numtokens": 256}, "config": {...}},
...,
]
Classes
-------
- ConfigSet: In-memory collection of configs for a kernel with lookup/query APIs.
- ConfigManager: File-level operations for config persistence.
Config keys are ``CaseKey`` instances mapping parameter names to
values. The default config uses ``CaseKey.default()``.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
@@ -45,11 +45,13 @@ logger = init_logger(__name__)
class ConfigSet:
"""In-memory collection of Helion configs with lookup/query capabilities."""
"""In-memory collection of Helion configs with lookup/query capabilities.
# Type alias for nested config structure:
# platform -> config_key -> helion.Config
_ConfigDict = dict[str, dict[str, "helion.Config"]]
Configs are stored keyed by ``CaseKey``. The default config
uses ``CaseKey.default()`` as its key.
"""
_ConfigDict = dict[str, dict[CaseKey, "helion.Config"]]
def __init__(self, kernel_name: str):
self._kernel_name = kernel_name
@@ -59,7 +61,7 @@ class ConfigSet:
def kernel_name(self) -> str:
return self._kernel_name
def get_config(self, platform: str, config_key: str) -> helion.Config:
def get_config(self, platform: str, config_key: CaseKey) -> helion.Config:
platform_dict = self._configs.get(platform)
if platform_dict is None:
avail_platforms = self.get_platforms()
@@ -82,7 +84,8 @@ class ConfigSet:
avail_keys = self.get_config_keys(platform)
raise KeyError(
f"Config not found for kernel '{self._kernel_name}': "
f"config_key '{config_key}' not found for platform '{platform}'. "
f"config_key '{config_key}' not found for "
f"platform '{platform}'. "
f"Available config_keys: {avail_keys or '(none)'}"
)
@@ -91,25 +94,34 @@ class ConfigSet:
def get_platforms(self) -> list[str]:
return sorted(self._configs.keys())
def get_config_keys(self, platform: str) -> list[str]:
def get_config_keys(self, platform: str) -> list[CaseKey]:
platform_dict = self._configs.get(platform.lower())
if platform_dict is None:
return []
return sorted(platform_dict.keys())
return sorted(platform_dict.keys(), key=str)
def to_dict(self) -> dict[str, Any]:
result: dict[str, Any] = {}
for platform, config_keys_dict in self._configs.items():
result[platform] = {}
for config_key, config in config_keys_dict.items():
result[platform][config_key] = json.loads(config.to_json())
def to_config_entries(self) -> dict[str, list[dict[str, Any]]]:
"""Serialize to config entries format for JSON output."""
result: dict[str, list[dict[str, Any]]] = {}
for platform, config_dict in self._configs.items():
pairs: list[dict[str, Any]] = []
for config_key, config in config_dict.items():
config_data = json.loads(config.to_json())
pairs.append({"key": dict(config_key), "config": config_data})
result[platform] = pairs
return result
def to_dict(self) -> dict[str, dict[CaseKey, Any]]:
"""Return configs as a nested dict (platform -> key -> config)."""
result: dict[str, dict[CaseKey, Any]] = {}
for platform, config_dict in self._configs.items():
result[platform] = {
k: json.loads(v.to_json()) for k, v in config_dict.items()
}
return result
@classmethod
def from_dict(cls, kernel_name: str, data: dict[str, Any]) -> "ConfigSet":
def from_dict(cls, kernel_name: str, data: dict[str, Any]) -> ConfigSet:
config_set = cls(kernel_name)
count = 0
@@ -117,9 +129,11 @@ class ConfigSet:
if platform not in config_set._configs:
config_set._configs[platform] = {}
for config_key, config_data in platform_data.items():
config = helion.Config(**config_data)
config_set._configs[platform][config_key] = config
for entry in platform_data:
raw_key = entry["key"]
key = CaseKey.default() if not raw_key else CaseKey(raw_key)
config = helion.Config(**entry["config"])
config_set._configs[platform][key] = config
count += 1
if count > 0:
@@ -132,7 +146,10 @@ class ConfigSet:
return config_set
def set_config(
self, platform: str, config_key: str, config: "helion.Config"
self,
platform: str,
config_key: CaseKey,
config: helion.Config,
) -> None:
platform = platform.lower()
if platform not in self._configs:
@@ -145,7 +162,7 @@ class ConfigSet:
config_key,
)
def has_config(self, platform: str, config_key: str) -> bool:
def has_config(self, platform: str, config_key: CaseKey) -> bool:
platform = platform.lower()
platform_dict = self._configs.get(platform)
if platform_dict is None:
@@ -156,18 +173,18 @@ class ConfigSet:
class ConfigManager:
"""File-level configuration management for Helion kernels (global singleton)."""
_instance: "ConfigManager | None" = None
_instance: ConfigManager | None = None
_instance_base_dir: Path | None = None
def __new__(cls, base_dir: str | Path | None = None) -> "ConfigManager":
def __new__(cls, base_dir: str | Path | None = None) -> ConfigManager:
resolved_base_dir = cls._resolve_base_dir(base_dir)
if cls._instance is not None:
if cls._instance_base_dir != resolved_base_dir:
raise ValueError(
f"ConfigManager singleton already exists with base_dir "
f"'{cls._instance_base_dir}', cannot create with different "
f"base_dir '{resolved_base_dir}'"
f"'{cls._instance_base_dir}', cannot create with "
f"different base_dir '{resolved_base_dir}'"
)
return cls._instance
@@ -190,7 +207,7 @@ class ConfigManager:
return (Path(__file__).parent / "configs").resolve()
@classmethod
def get_instance(cls) -> "ConfigManager":
def get_instance(cls) -> ConfigManager:
if cls._instance is None:
raise RuntimeError(
"ConfigManager instance has not been created. "
@@ -229,16 +246,16 @@ class ConfigManager:
f"Config directory '{self._base_dir}' is not writable: {e}"
) from e
def _load_platform_file(self, kernel_name: str, platform: str) -> dict[str, Any]:
def _load_platform_file(self, kernel_name: str, platform: str) -> Any:
config_path = self.get_config_file_path(kernel_name, platform)
if not config_path.exists():
return {}
return []
try:
with open(config_path) as f:
return json.load(f)
except (json.JSONDecodeError, OSError) as e:
logger.error("Failed to load config file %s: %s", config_path, e)
return {}
return []
def load_config_set(self, kernel_name: str) -> ConfigSet:
kernel_dir = self.get_kernel_dir(kernel_name)
@@ -253,32 +270,36 @@ class ConfigManager:
platform_data = json.load(f)
data[platform] = platform_data
except (json.JSONDecodeError, OSError) as e:
logger.error("Failed to load config file %s: %s", platform_file, e)
logger.error(
"Failed to load config file %s: %s",
platform_file,
e,
)
return ConfigSet.from_dict(kernel_name, data)
def get_platform_configs(
self, kernel_name: str, platform: str
) -> dict[str, helion.Config]:
) -> dict[CaseKey, helion.Config]:
platform_data = self._load_platform_file(kernel_name, platform)
if not platform_data:
return {}
config_set = ConfigSet.from_dict(kernel_name, {platform: platform_data})
config_keys = config_set.get_config_keys(platform)
return {
config_key: config_set.get_config(platform, config_key)
for config_key in config_keys
k: config_set.get_config(platform, k)
for k in config_set.get_config_keys(platform)
}
def save_config_set(self, config_set: ConfigSet) -> Path:
kernel_dir = self.get_kernel_dir(config_set.kernel_name)
kernel_dir.mkdir(parents=True, exist_ok=True)
full_data = config_set.to_dict()
for platform, platform_data in full_data.items():
full_data = config_set.to_config_entries()
for platform, pairs in full_data.items():
platform_path = kernel_dir / f"{platform}.json"
with open(platform_path, "w") as f:
json.dump(platform_data, f, indent=2)
json.dump(pairs, f, indent=2)
f.write("\n")
logger.info("Saved config to: %s", platform_path)
return kernel_dir
@@ -287,21 +308,34 @@ class ConfigManager:
self,
kernel_name: str,
platform: str,
configs: dict[str, "helion.Config"],
configs: dict[CaseKey, helion.Config],
) -> Path:
"""Save configs for a kernel/platform, merging with existing."""
platform_data = self._load_platform_file(kernel_name, platform)
for config_key, config in configs.items():
platform_data[config_key] = json.loads(config.to_json())
config_set = ConfigSet.from_dict(
kernel_name,
{platform: self._load_platform_file(kernel_name, platform)},
)
for key, config in configs.items():
config_set.set_config(platform, key, config)
pairs = config_set.to_config_entries().get(platform, [])
platform_path = self.get_config_file_path(kernel_name, platform)
platform_path.parent.mkdir(parents=True, exist_ok=True)
with open(platform_path, "w") as f:
json.dump(platform_data, f, indent=2)
json.dump(pairs, f, indent=2)
f.write("\n")
logger.info("Saved config to: %s", platform_path)
return platform_path
def config_exists(self, kernel_name: str, platform: str, config_key: str) -> bool:
def config_exists(
self,
kernel_name: str,
platform: str,
config_key: CaseKey,
) -> bool:
platform_data = self._load_platform_file(kernel_name, platform)
return config_key in platform_data
if not platform_data:
return False
target = dict(config_key)
return any(entry["key"] == target for entry in platform_data)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+32 -31
View File
@@ -1,11 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from typing import Any
import regex as re
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
@@ -22,14 +24,14 @@ from vllm.kernels.helion.register import register_kernel
logger = init_logger(__name__)
def generate_silu_mul_fp8_inputs() -> dict[str, tuple[Any, ...]]:
def generate_silu_mul_fp8_inputs() -> dict[CaseKey, tuple[Any, ...]]:
intermediate_sizes = [2048, 2880, 4096, 8192, 11008, 14336]
# Use the same num_tokens values as vLLM's default cudagraph capture sizes.
# See vllm/config/vllm.py _set_cudagraph_sizes() for the canonical formula.
num_tokens_list = [1, 2, 4] + list(range(8, 256, 8)) + list(range(256, 513, 16))
inputs = {}
inputs: dict[CaseKey, tuple[Any, ...]] = {}
for num_tokens in num_tokens_list:
for intermediate_size in intermediate_sizes:
input_tensor = torch.randn(
@@ -40,15 +42,18 @@ def generate_silu_mul_fp8_inputs() -> dict[str, tuple[Any, ...]]:
)
scale = torch.tensor([1.0], device="cuda", dtype=torch.float32)
config_key = f"intermediate_{intermediate_size}_numtokens_{num_tokens}"
inputs[config_key] = (input_tensor, scale)
key = CaseKey({"intermediate": intermediate_size, "numtokens": num_tokens})
inputs[key] = (input_tensor, scale)
return inputs
_pick_cache: dict[tuple[int, int], CaseKey | None] = {}
def pick_silu_mul_fp8_config(
args: tuple[Any, ...], config_keys: list[str]
) -> str | None:
args: tuple[Any, ...], config_keys: list[CaseKey]
) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
@@ -57,39 +62,35 @@ def pick_silu_mul_fp8_config(
2. Among the num_tokens values tuned for that intermediate_size, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
Config keys must be "default" or follow the format
"intermediate_{int}_numtokens_{int}".
"""
if not config_keys:
return None
input_tensor, _scale = args
intermediate_size = input_tensor.shape[-1] // 2
num_tokens = input_tensor.view(-1, input_tensor.shape[-1]).shape[0]
configs: dict[int, list[int]] = {}
for key in config_keys:
if key == "default":
intermediate_size = int(input_tensor.shape[-1]) // 2
num_tokens = int(input_tensor.view(-1, input_tensor.shape[-1]).shape[0])
cache_key = (num_tokens, intermediate_size)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
by_isize: dict[int, list[int]] = {}
for k in config_keys:
if k.is_default():
continue
match = re.fullmatch(r"intermediate_(\d+)_numtokens_(\d+)", key)
if not match:
raise ValueError(
f"Malformed config key '{key}', "
f"expected format 'intermediate_{{int}}_numtokens_{{int}}'"
)
isize_str, ntokens_str = match.groups()
configs.setdefault(int(isize_str), []).append(int(ntokens_str))
by_isize.setdefault(k["intermediate"], []).append(k["numtokens"])
if not configs:
return "default" if "default" in config_keys else None
if not by_isize:
return None
best_isize = min(configs, key=lambda s: abs(s - intermediate_size))
available_ntokens = sorted(configs[best_isize])
best_ntokens = next(
(n for n in available_ntokens if n >= num_tokens), available_ntokens[-1]
)
best_isize = min(by_isize, key=lambda s: abs(s - intermediate_size))
available = sorted(by_isize[best_isize])
best_ntokens = next((n for n in available if n >= num_tokens), available[-1])
return f"intermediate_{best_isize}_numtokens_{best_ntokens}"
result = CaseKey({"intermediate": best_isize, "numtokens": best_ntokens})
_pick_cache[cache_key] = result
return result
@register_kernel(
+50 -50
View File
@@ -36,12 +36,15 @@ Key Classes
- PresetConfigSearch: Custom autotuner that returns pre-tuned configs
"""
from __future__ import annotations
from collections.abc import Callable
from typing import Any, cast
from typing import Any
import torch
from torch.library import Library
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
from vllm.utils.torch_utils import direct_register_custom_op
@@ -76,9 +79,11 @@ logger = init_logger(__name__)
vllm_helion_lib = Library("vllm_helion", "FRAGMENT") # noqa
ConfigPicker = Callable[[tuple[Any, ...], list[CaseKey]], CaseKey | None]
def validate_helion_settings(
helion_settings: "helion.Settings | None", op_name: str
helion_settings: helion.Settings | None, op_name: str
) -> None:
if helion_settings is None:
return
@@ -107,7 +112,7 @@ def validate_helion_settings(
def create_helion_decorated_kernel(
raw_kernel_func: Callable,
helion_settings: "helion.Settings | None" = None,
helion_settings: helion.Settings | None = None,
extra_kwargs: dict[str, Any] | None = None,
) -> Any:
kernel_kwargs: dict[str, Any] = {}
@@ -144,9 +149,9 @@ class ConfiguredHelionKernel:
def __init__(
self,
op_name: str,
config_picker: Callable[[tuple[Any, ...], list[str]], str | None] | None,
config_picker: ConfigPicker | None,
raw_kernel_func: Callable,
helion_settings: "helion.Settings | None" = None,
helion_settings: helion.Settings | None = None,
):
self.op_name = op_name
self.config_picker = config_picker
@@ -170,41 +175,44 @@ class ConfiguredHelionKernel:
f"A config_picker must be provided to register_kernel()."
)
# After None check, config_picker is guaranteed to be non-None
assert self.config_picker is not None
picker = self.config_picker
all_keys = list(self.configs.keys())
default = CaseKey.default()
has_default = default in self.configs
def key_computer(*args):
config_keys = list(self.configs.keys())
# Cast is safe because we checked for None above
config_picker = cast(
Callable[[tuple[Any, ...], list[str]], str | None], self.config_picker
)
selected_key = config_picker(args, config_keys)
if selected_key:
return selected_key
return "default" if "default" in self.configs else None
selected = picker(args, all_keys)
if selected is not None:
return str(selected)
if has_default:
return str(default)
return None
return key_computer
def _create_config_selector(self, key_computer):
def config_selector(args):
# args is a tuple; key_computer expects unpacked args
selected_config_key = key_computer(*args)
str_to_key = {str(k): k for k in self.configs}
if selected_config_key is None:
def config_selector(args):
selected_str = key_computer(*args)
if selected_str is None:
raise ValueError(
f"Config picker returned None for kernel '{self.op_name}' "
f"with available config keys: {list(self.configs.keys())}"
f"Config picker returned None for kernel "
f"'{self.op_name}' with available config keys: "
f"{list(self.configs.keys())}"
)
if selected_config_key not in self.configs:
config_key = str_to_key.get(selected_str)
if config_key is None:
raise ValueError(
f"Config picker returned invalid config key "
f"'{selected_config_key}' for kernel '{self.op_name}'. "
f"'{selected_str}' for kernel "
f"'{self.op_name}'. "
f"Available keys: {list(self.configs.keys())}"
)
return self.configs[selected_config_key]
return self.configs[config_key]
return config_selector
@@ -251,9 +259,9 @@ class HelionKernelWrapper:
raw_kernel_func: Callable,
op_name: str,
fake_impl: Callable,
config_picker: Callable[[tuple[Any, ...], list[str]], str | None],
helion_settings: "helion.Settings | None" = None,
input_generator: Callable[[], dict[str, tuple[Any, ...]]] | None = None,
config_picker: ConfigPicker,
helion_settings: helion.Settings | None = None,
input_generator: (Callable[[], dict[CaseKey, tuple[Any, ...]]] | None) = None,
):
# Validate helion_settings doesn't conflict with our custom autotuner
validate_helion_settings(helion_settings, op_name)
@@ -302,7 +310,7 @@ class HelionKernelWrapper:
# During eager execution, call the kernel directly.
return self._configured_kernel(*args, **kwargs)
def get_inputs(self) -> dict[str, tuple[Any, ...]]:
def get_inputs(self) -> dict[CaseKey, tuple[Any, ...]]:
if self._input_generator is None:
raise NotImplementedError(
f"No input generator registered for kernel '{self.op_name}'. "
@@ -370,7 +378,7 @@ def get_kernel_by_name(kernel_name: str) -> HelionKernelWrapper | None:
def infer_fake_impl(
kernel_func: Callable,
helion_settings: "helion.Settings | None" = None,
helion_settings: helion.Settings | None = None,
) -> Callable:
def helion_fake_kernel(*args, **kwargs):
kernel_kwargs = {}
@@ -392,37 +400,29 @@ def infer_fake_impl(
def register_kernel(
op_name: str | None = None,
*,
config_picker: Callable[[tuple[Any, ...], list[str]], str | None],
config_picker: ConfigPicker,
fake_impl: Callable | None = None,
helion_settings: "helion.Settings | None" = None,
input_generator: Callable[[], dict[str, tuple[Any, ...]]] | None = None,
helion_settings: helion.Settings | None = None,
input_generator: (Callable[[], dict[CaseKey, tuple[Any, ...]]] | None) = None,
) -> Callable[[Callable], HelionKernelWrapper]:
"""Register a Helion kernel with pre-tuned config selection.
Wraps the kernel function in a HelionKernelWrapper that eagerly builds
the configured kernel and (on older PyTorch) registers a custom op.
Args:
config_picker: Required. Function with signature
``(args: tuple, config_keys: list[str]) -> str | None``
that picks the best config key from available options.
Return ``None`` to fall back to ``"default"``.
config_picker: Required. Receives ``(args, config_keys)``
where each config key is a ``dict[str, Any]`` mapping
parameter names to values. Return the best-matching
dict, or ``None`` to fall back to the default config.
Example::
def pick_config(args, config_keys):
x = args[0]
hidden_size = x.shape[-1]
batch_size = x.shape[0]
for key in config_keys:
if key == f"hiddensize_{hidden_size}_batchsize_{batch_size}":
return key
return "default" if "default" in config_keys else None
best = min(config_keys, key=lambda k: abs(k["size"] - x.shape[0]))
return best
input_generator: Optional. Function that returns
``dict[str, tuple]`` where each key is a configuration
identifier (e.g. ``"4096"``, ``"hidden_4096"``) and each
value is a tuple of arguments to pass to the kernel.
input_generator: Optional. Returns ``dict[str, tuple]`` where
each key is a serialized config key and each value is a
tuple of arguments to pass to the kernel.
Example::