[Kernel][Helion] Optimize Helion config parsing latency (#40850)
Signed-off-by: Yanan Cao <gmagogsfm@gmail.com> Co-authored-by: Claude Sonnet 4 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4
parent
baf068d8be
commit
0b99971352
@@ -3,11 +3,13 @@
|
||||
"""Helion integration for vLLM."""
|
||||
|
||||
import vllm.kernels.helion.ops # noqa: F401 Auto-register all Helion ops
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import (
|
||||
ConfigManager,
|
||||
ConfigSet,
|
||||
)
|
||||
from vllm.kernels.helion.register import (
|
||||
ConfigPicker,
|
||||
ConfiguredHelionKernel,
|
||||
HelionKernelWrapper,
|
||||
get_kernel_by_name,
|
||||
@@ -19,9 +21,11 @@ from vllm.kernels.helion.utils import canonicalize_gpu_name, get_canonical_gpu_n
|
||||
|
||||
__all__ = [
|
||||
# Config management
|
||||
"CaseKey",
|
||||
"ConfigManager",
|
||||
"ConfigSet",
|
||||
# Kernel registration
|
||||
"ConfigPicker",
|
||||
"ConfiguredHelionKernel",
|
||||
"HelionKernelWrapper",
|
||||
"get_kernel_by_name",
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Structured key for identifying kernel config/autotune/benchmark cases.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
|
||||
class CaseKey(dict[str, Any]):
|
||||
"""Immutable, hashable dict for identifying kernel cases.
|
||||
|
||||
Used as the key for config lookup, autotuning, benchmarking, and
|
||||
input generation. Behaves like a read-only dict and can be used
|
||||
as a dict key or in sets.
|
||||
|
||||
The canonical string form (``__str__``) is stable JSON with sorted
|
||||
keys. Use ``CaseKey.default()`` for the default/fallback key.
|
||||
The regular constructor requires at least one key-value pair::
|
||||
|
||||
CaseKey({"intermediate": 2048, "numtokens": 256})
|
||||
CaseKey.default() # default/fallback
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, _allow_empty: bool = False, **kwargs: Any):
|
||||
super().__init__(*args, **kwargs)
|
||||
if not self and not _allow_empty:
|
||||
raise TypeError(
|
||||
"CaseKey requires at least one key-value pair. "
|
||||
"Use CaseKey.default() for the default config key."
|
||||
)
|
||||
self._str: str | None = None
|
||||
self._hash: int | None = None
|
||||
|
||||
@classmethod
|
||||
def default(cls) -> CaseKey:
|
||||
"""Create a default case key (empty)."""
|
||||
return cls(_allow_empty=True)
|
||||
|
||||
def __hash__(self) -> int: # type: ignore[override]
|
||||
if self._hash is None:
|
||||
self._hash = hash(str(self))
|
||||
return self._hash
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self._str is None:
|
||||
self._str = json.dumps(dict(self), sort_keys=True, separators=(",", ":"))
|
||||
return self._str
|
||||
|
||||
def __repr__(self) -> str:
|
||||
if not self:
|
||||
return "CaseKey.default()"
|
||||
return f"CaseKey({dict(self)})"
|
||||
|
||||
def is_default(self) -> bool:
|
||||
"""Return True if this is the default case key (empty)."""
|
||||
return not self
|
||||
|
||||
def _readonly(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise TypeError("CaseKey is immutable")
|
||||
|
||||
__setitem__ = _readonly # type: ignore[assignment]
|
||||
__delitem__ = _readonly # type: ignore[assignment]
|
||||
__ior__ = _readonly # type: ignore[assignment]
|
||||
update = _readonly # type: ignore[assignment]
|
||||
pop = _readonly # type: ignore[assignment]
|
||||
popitem = _readonly # type: ignore[assignment]
|
||||
setdefault = _readonly # type: ignore[assignment]
|
||||
clear = _readonly # type: ignore[assignment]
|
||||
@@ -11,25 +11,25 @@ Config File Structure
|
||||
Each kernel has a directory: {kernel_name}/
|
||||
Inside, each GPU platform has its own JSON file: {kernel_name}/{platform}.json
|
||||
|
||||
For example:
|
||||
silu_mul_fp8/
|
||||
nvidia_h100.json # { "default": {...}, "batch_32_hidden_4096": {...} }
|
||||
nvidia_h200.json # { "batch_16_hidden_2048": {...} }
|
||||
Platform files store config entries as a JSON array::
|
||||
|
||||
Each platform file maps config keys to Helion config objects.
|
||||
Config keys should be structured strings that encode the relevant
|
||||
parameters (e.g., "batch_32_hidden_4096", "seq_512_heads_16", "fp8_batch_64", etc.).
|
||||
[
|
||||
{"key": {}, "config": {...}},
|
||||
{"key": {"intermediate": 2048, "numtokens": 256}, "config": {...}},
|
||||
...,
|
||||
]
|
||||
|
||||
Classes
|
||||
-------
|
||||
- ConfigSet: In-memory collection of configs for a kernel with lookup/query APIs.
|
||||
- ConfigManager: File-level operations for config persistence.
|
||||
Config keys are ``CaseKey`` instances mapping parameter names to
|
||||
values. The default config uses ``CaseKey.default()``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
@@ -45,11 +45,13 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ConfigSet:
|
||||
"""In-memory collection of Helion configs with lookup/query capabilities."""
|
||||
"""In-memory collection of Helion configs with lookup/query capabilities.
|
||||
|
||||
# Type alias for nested config structure:
|
||||
# platform -> config_key -> helion.Config
|
||||
_ConfigDict = dict[str, dict[str, "helion.Config"]]
|
||||
Configs are stored keyed by ``CaseKey``. The default config
|
||||
uses ``CaseKey.default()`` as its key.
|
||||
"""
|
||||
|
||||
_ConfigDict = dict[str, dict[CaseKey, "helion.Config"]]
|
||||
|
||||
def __init__(self, kernel_name: str):
|
||||
self._kernel_name = kernel_name
|
||||
@@ -59,7 +61,7 @@ class ConfigSet:
|
||||
def kernel_name(self) -> str:
|
||||
return self._kernel_name
|
||||
|
||||
def get_config(self, platform: str, config_key: str) -> helion.Config:
|
||||
def get_config(self, platform: str, config_key: CaseKey) -> helion.Config:
|
||||
platform_dict = self._configs.get(platform)
|
||||
if platform_dict is None:
|
||||
avail_platforms = self.get_platforms()
|
||||
@@ -82,7 +84,8 @@ class ConfigSet:
|
||||
avail_keys = self.get_config_keys(platform)
|
||||
raise KeyError(
|
||||
f"Config not found for kernel '{self._kernel_name}': "
|
||||
f"config_key '{config_key}' not found for platform '{platform}'. "
|
||||
f"config_key '{config_key}' not found for "
|
||||
f"platform '{platform}'. "
|
||||
f"Available config_keys: {avail_keys or '(none)'}"
|
||||
)
|
||||
|
||||
@@ -91,25 +94,34 @@ class ConfigSet:
|
||||
def get_platforms(self) -> list[str]:
|
||||
return sorted(self._configs.keys())
|
||||
|
||||
def get_config_keys(self, platform: str) -> list[str]:
|
||||
def get_config_keys(self, platform: str) -> list[CaseKey]:
|
||||
platform_dict = self._configs.get(platform.lower())
|
||||
if platform_dict is None:
|
||||
return []
|
||||
return sorted(platform_dict.keys())
|
||||
return sorted(platform_dict.keys(), key=str)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
|
||||
for platform, config_keys_dict in self._configs.items():
|
||||
result[platform] = {}
|
||||
|
||||
for config_key, config in config_keys_dict.items():
|
||||
result[platform][config_key] = json.loads(config.to_json())
|
||||
def to_config_entries(self) -> dict[str, list[dict[str, Any]]]:
|
||||
"""Serialize to config entries format for JSON output."""
|
||||
result: dict[str, list[dict[str, Any]]] = {}
|
||||
for platform, config_dict in self._configs.items():
|
||||
pairs: list[dict[str, Any]] = []
|
||||
for config_key, config in config_dict.items():
|
||||
config_data = json.loads(config.to_json())
|
||||
pairs.append({"key": dict(config_key), "config": config_data})
|
||||
result[platform] = pairs
|
||||
return result
|
||||
|
||||
def to_dict(self) -> dict[str, dict[CaseKey, Any]]:
|
||||
"""Return configs as a nested dict (platform -> key -> config)."""
|
||||
result: dict[str, dict[CaseKey, Any]] = {}
|
||||
for platform, config_dict in self._configs.items():
|
||||
result[platform] = {
|
||||
k: json.loads(v.to_json()) for k, v in config_dict.items()
|
||||
}
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, kernel_name: str, data: dict[str, Any]) -> "ConfigSet":
|
||||
def from_dict(cls, kernel_name: str, data: dict[str, Any]) -> ConfigSet:
|
||||
config_set = cls(kernel_name)
|
||||
count = 0
|
||||
|
||||
@@ -117,9 +129,11 @@ class ConfigSet:
|
||||
if platform not in config_set._configs:
|
||||
config_set._configs[platform] = {}
|
||||
|
||||
for config_key, config_data in platform_data.items():
|
||||
config = helion.Config(**config_data)
|
||||
config_set._configs[platform][config_key] = config
|
||||
for entry in platform_data:
|
||||
raw_key = entry["key"]
|
||||
key = CaseKey.default() if not raw_key else CaseKey(raw_key)
|
||||
config = helion.Config(**entry["config"])
|
||||
config_set._configs[platform][key] = config
|
||||
count += 1
|
||||
|
||||
if count > 0:
|
||||
@@ -132,7 +146,10 @@ class ConfigSet:
|
||||
return config_set
|
||||
|
||||
def set_config(
|
||||
self, platform: str, config_key: str, config: "helion.Config"
|
||||
self,
|
||||
platform: str,
|
||||
config_key: CaseKey,
|
||||
config: helion.Config,
|
||||
) -> None:
|
||||
platform = platform.lower()
|
||||
if platform not in self._configs:
|
||||
@@ -145,7 +162,7 @@ class ConfigSet:
|
||||
config_key,
|
||||
)
|
||||
|
||||
def has_config(self, platform: str, config_key: str) -> bool:
|
||||
def has_config(self, platform: str, config_key: CaseKey) -> bool:
|
||||
platform = platform.lower()
|
||||
platform_dict = self._configs.get(platform)
|
||||
if platform_dict is None:
|
||||
@@ -156,18 +173,18 @@ class ConfigSet:
|
||||
class ConfigManager:
|
||||
"""File-level configuration management for Helion kernels (global singleton)."""
|
||||
|
||||
_instance: "ConfigManager | None" = None
|
||||
_instance: ConfigManager | None = None
|
||||
_instance_base_dir: Path | None = None
|
||||
|
||||
def __new__(cls, base_dir: str | Path | None = None) -> "ConfigManager":
|
||||
def __new__(cls, base_dir: str | Path | None = None) -> ConfigManager:
|
||||
resolved_base_dir = cls._resolve_base_dir(base_dir)
|
||||
|
||||
if cls._instance is not None:
|
||||
if cls._instance_base_dir != resolved_base_dir:
|
||||
raise ValueError(
|
||||
f"ConfigManager singleton already exists with base_dir "
|
||||
f"'{cls._instance_base_dir}', cannot create with different "
|
||||
f"base_dir '{resolved_base_dir}'"
|
||||
f"'{cls._instance_base_dir}', cannot create with "
|
||||
f"different base_dir '{resolved_base_dir}'"
|
||||
)
|
||||
return cls._instance
|
||||
|
||||
@@ -190,7 +207,7 @@ class ConfigManager:
|
||||
return (Path(__file__).parent / "configs").resolve()
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls) -> "ConfigManager":
|
||||
def get_instance(cls) -> ConfigManager:
|
||||
if cls._instance is None:
|
||||
raise RuntimeError(
|
||||
"ConfigManager instance has not been created. "
|
||||
@@ -229,16 +246,16 @@ class ConfigManager:
|
||||
f"Config directory '{self._base_dir}' is not writable: {e}"
|
||||
) from e
|
||||
|
||||
def _load_platform_file(self, kernel_name: str, platform: str) -> dict[str, Any]:
|
||||
def _load_platform_file(self, kernel_name: str, platform: str) -> Any:
|
||||
config_path = self.get_config_file_path(kernel_name, platform)
|
||||
if not config_path.exists():
|
||||
return {}
|
||||
return []
|
||||
try:
|
||||
with open(config_path) as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.error("Failed to load config file %s: %s", config_path, e)
|
||||
return {}
|
||||
return []
|
||||
|
||||
def load_config_set(self, kernel_name: str) -> ConfigSet:
|
||||
kernel_dir = self.get_kernel_dir(kernel_name)
|
||||
@@ -253,32 +270,36 @@ class ConfigManager:
|
||||
platform_data = json.load(f)
|
||||
data[platform] = platform_data
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.error("Failed to load config file %s: %s", platform_file, e)
|
||||
logger.error(
|
||||
"Failed to load config file %s: %s",
|
||||
platform_file,
|
||||
e,
|
||||
)
|
||||
|
||||
return ConfigSet.from_dict(kernel_name, data)
|
||||
|
||||
def get_platform_configs(
|
||||
self, kernel_name: str, platform: str
|
||||
) -> dict[str, helion.Config]:
|
||||
) -> dict[CaseKey, helion.Config]:
|
||||
platform_data = self._load_platform_file(kernel_name, platform)
|
||||
if not platform_data:
|
||||
return {}
|
||||
config_set = ConfigSet.from_dict(kernel_name, {platform: platform_data})
|
||||
config_keys = config_set.get_config_keys(platform)
|
||||
return {
|
||||
config_key: config_set.get_config(platform, config_key)
|
||||
for config_key in config_keys
|
||||
k: config_set.get_config(platform, k)
|
||||
for k in config_set.get_config_keys(platform)
|
||||
}
|
||||
|
||||
def save_config_set(self, config_set: ConfigSet) -> Path:
|
||||
kernel_dir = self.get_kernel_dir(config_set.kernel_name)
|
||||
kernel_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
full_data = config_set.to_dict()
|
||||
for platform, platform_data in full_data.items():
|
||||
full_data = config_set.to_config_entries()
|
||||
for platform, pairs in full_data.items():
|
||||
platform_path = kernel_dir / f"{platform}.json"
|
||||
with open(platform_path, "w") as f:
|
||||
json.dump(platform_data, f, indent=2)
|
||||
json.dump(pairs, f, indent=2)
|
||||
f.write("\n")
|
||||
logger.info("Saved config to: %s", platform_path)
|
||||
|
||||
return kernel_dir
|
||||
@@ -287,21 +308,34 @@ class ConfigManager:
|
||||
self,
|
||||
kernel_name: str,
|
||||
platform: str,
|
||||
configs: dict[str, "helion.Config"],
|
||||
configs: dict[CaseKey, helion.Config],
|
||||
) -> Path:
|
||||
"""Save configs for a kernel/platform, merging with existing."""
|
||||
platform_data = self._load_platform_file(kernel_name, platform)
|
||||
for config_key, config in configs.items():
|
||||
platform_data[config_key] = json.loads(config.to_json())
|
||||
config_set = ConfigSet.from_dict(
|
||||
kernel_name,
|
||||
{platform: self._load_platform_file(kernel_name, platform)},
|
||||
)
|
||||
for key, config in configs.items():
|
||||
config_set.set_config(platform, key, config)
|
||||
|
||||
pairs = config_set.to_config_entries().get(platform, [])
|
||||
platform_path = self.get_config_file_path(kernel_name, platform)
|
||||
platform_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(platform_path, "w") as f:
|
||||
json.dump(platform_data, f, indent=2)
|
||||
json.dump(pairs, f, indent=2)
|
||||
f.write("\n")
|
||||
|
||||
logger.info("Saved config to: %s", platform_path)
|
||||
return platform_path
|
||||
|
||||
def config_exists(self, kernel_name: str, platform: str, config_key: str) -> bool:
|
||||
def config_exists(
|
||||
self,
|
||||
kernel_name: str,
|
||||
platform: str,
|
||||
config_key: CaseKey,
|
||||
) -> bool:
|
||||
platform_data = self._load_platform_file(kernel_name, platform)
|
||||
return config_key in platform_data
|
||||
if not platform_data:
|
||||
return False
|
||||
target = dict(config_key)
|
||||
return any(entry["key"] == target for entry in platform_data)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,11 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
@@ -22,14 +24,14 @@ from vllm.kernels.helion.register import register_kernel
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def generate_silu_mul_fp8_inputs() -> dict[str, tuple[Any, ...]]:
|
||||
def generate_silu_mul_fp8_inputs() -> dict[CaseKey, tuple[Any, ...]]:
|
||||
intermediate_sizes = [2048, 2880, 4096, 8192, 11008, 14336]
|
||||
|
||||
# Use the same num_tokens values as vLLM's default cudagraph capture sizes.
|
||||
# See vllm/config/vllm.py _set_cudagraph_sizes() for the canonical formula.
|
||||
num_tokens_list = [1, 2, 4] + list(range(8, 256, 8)) + list(range(256, 513, 16))
|
||||
|
||||
inputs = {}
|
||||
inputs: dict[CaseKey, tuple[Any, ...]] = {}
|
||||
for num_tokens in num_tokens_list:
|
||||
for intermediate_size in intermediate_sizes:
|
||||
input_tensor = torch.randn(
|
||||
@@ -40,15 +42,18 @@ def generate_silu_mul_fp8_inputs() -> dict[str, tuple[Any, ...]]:
|
||||
)
|
||||
scale = torch.tensor([1.0], device="cuda", dtype=torch.float32)
|
||||
|
||||
config_key = f"intermediate_{intermediate_size}_numtokens_{num_tokens}"
|
||||
inputs[config_key] = (input_tensor, scale)
|
||||
key = CaseKey({"intermediate": intermediate_size, "numtokens": num_tokens})
|
||||
inputs[key] = (input_tensor, scale)
|
||||
|
||||
return inputs
|
||||
|
||||
|
||||
_pick_cache: dict[tuple[int, int], CaseKey | None] = {}
|
||||
|
||||
|
||||
def pick_silu_mul_fp8_config(
|
||||
args: tuple[Any, ...], config_keys: list[str]
|
||||
) -> str | None:
|
||||
args: tuple[Any, ...], config_keys: list[CaseKey]
|
||||
) -> CaseKey | None:
|
||||
"""Pick the best pre-tuned config for the given input shape.
|
||||
|
||||
Selection strategy:
|
||||
@@ -57,39 +62,35 @@ def pick_silu_mul_fp8_config(
|
||||
2. Among the num_tokens values tuned for that intermediate_size, pick
|
||||
the smallest num_tokens >= the input's num_tokens. If the input is
|
||||
larger than all available num_tokens, fall back to the largest.
|
||||
|
||||
Config keys must be "default" or follow the format
|
||||
"intermediate_{int}_numtokens_{int}".
|
||||
"""
|
||||
if not config_keys:
|
||||
return None
|
||||
|
||||
input_tensor, _scale = args
|
||||
intermediate_size = input_tensor.shape[-1] // 2
|
||||
num_tokens = input_tensor.view(-1, input_tensor.shape[-1]).shape[0]
|
||||
configs: dict[int, list[int]] = {}
|
||||
for key in config_keys:
|
||||
if key == "default":
|
||||
intermediate_size = int(input_tensor.shape[-1]) // 2
|
||||
num_tokens = int(input_tensor.view(-1, input_tensor.shape[-1]).shape[0])
|
||||
|
||||
cache_key = (num_tokens, intermediate_size)
|
||||
cached = _pick_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
by_isize: dict[int, list[int]] = {}
|
||||
for k in config_keys:
|
||||
if k.is_default():
|
||||
continue
|
||||
match = re.fullmatch(r"intermediate_(\d+)_numtokens_(\d+)", key)
|
||||
if not match:
|
||||
raise ValueError(
|
||||
f"Malformed config key '{key}', "
|
||||
f"expected format 'intermediate_{{int}}_numtokens_{{int}}'"
|
||||
)
|
||||
isize_str, ntokens_str = match.groups()
|
||||
configs.setdefault(int(isize_str), []).append(int(ntokens_str))
|
||||
by_isize.setdefault(k["intermediate"], []).append(k["numtokens"])
|
||||
|
||||
if not configs:
|
||||
return "default" if "default" in config_keys else None
|
||||
if not by_isize:
|
||||
return None
|
||||
|
||||
best_isize = min(configs, key=lambda s: abs(s - intermediate_size))
|
||||
available_ntokens = sorted(configs[best_isize])
|
||||
best_ntokens = next(
|
||||
(n for n in available_ntokens if n >= num_tokens), available_ntokens[-1]
|
||||
)
|
||||
best_isize = min(by_isize, key=lambda s: abs(s - intermediate_size))
|
||||
available = sorted(by_isize[best_isize])
|
||||
best_ntokens = next((n for n in available if n >= num_tokens), available[-1])
|
||||
|
||||
return f"intermediate_{best_isize}_numtokens_{best_ntokens}"
|
||||
result = CaseKey({"intermediate": best_isize, "numtokens": best_ntokens})
|
||||
_pick_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
@register_kernel(
|
||||
|
||||
@@ -36,12 +36,15 @@ Key Classes
|
||||
- PresetConfigSearch: Custom autotuner that returns pre-tuned configs
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, cast
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch.library import Library
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import has_helion
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
@@ -76,9 +79,11 @@ logger = init_logger(__name__)
|
||||
|
||||
vllm_helion_lib = Library("vllm_helion", "FRAGMENT") # noqa
|
||||
|
||||
ConfigPicker = Callable[[tuple[Any, ...], list[CaseKey]], CaseKey | None]
|
||||
|
||||
|
||||
def validate_helion_settings(
|
||||
helion_settings: "helion.Settings | None", op_name: str
|
||||
helion_settings: helion.Settings | None, op_name: str
|
||||
) -> None:
|
||||
if helion_settings is None:
|
||||
return
|
||||
@@ -107,7 +112,7 @@ def validate_helion_settings(
|
||||
|
||||
def create_helion_decorated_kernel(
|
||||
raw_kernel_func: Callable,
|
||||
helion_settings: "helion.Settings | None" = None,
|
||||
helion_settings: helion.Settings | None = None,
|
||||
extra_kwargs: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
kernel_kwargs: dict[str, Any] = {}
|
||||
@@ -144,9 +149,9 @@ class ConfiguredHelionKernel:
|
||||
def __init__(
|
||||
self,
|
||||
op_name: str,
|
||||
config_picker: Callable[[tuple[Any, ...], list[str]], str | None] | None,
|
||||
config_picker: ConfigPicker | None,
|
||||
raw_kernel_func: Callable,
|
||||
helion_settings: "helion.Settings | None" = None,
|
||||
helion_settings: helion.Settings | None = None,
|
||||
):
|
||||
self.op_name = op_name
|
||||
self.config_picker = config_picker
|
||||
@@ -170,41 +175,44 @@ class ConfiguredHelionKernel:
|
||||
f"A config_picker must be provided to register_kernel()."
|
||||
)
|
||||
|
||||
# After None check, config_picker is guaranteed to be non-None
|
||||
assert self.config_picker is not None
|
||||
picker = self.config_picker
|
||||
all_keys = list(self.configs.keys())
|
||||
default = CaseKey.default()
|
||||
has_default = default in self.configs
|
||||
|
||||
def key_computer(*args):
|
||||
config_keys = list(self.configs.keys())
|
||||
# Cast is safe because we checked for None above
|
||||
config_picker = cast(
|
||||
Callable[[tuple[Any, ...], list[str]], str | None], self.config_picker
|
||||
)
|
||||
selected_key = config_picker(args, config_keys)
|
||||
if selected_key:
|
||||
return selected_key
|
||||
return "default" if "default" in self.configs else None
|
||||
selected = picker(args, all_keys)
|
||||
if selected is not None:
|
||||
return str(selected)
|
||||
if has_default:
|
||||
return str(default)
|
||||
return None
|
||||
|
||||
return key_computer
|
||||
|
||||
def _create_config_selector(self, key_computer):
|
||||
def config_selector(args):
|
||||
# args is a tuple; key_computer expects unpacked args
|
||||
selected_config_key = key_computer(*args)
|
||||
str_to_key = {str(k): k for k in self.configs}
|
||||
|
||||
if selected_config_key is None:
|
||||
def config_selector(args):
|
||||
selected_str = key_computer(*args)
|
||||
|
||||
if selected_str is None:
|
||||
raise ValueError(
|
||||
f"Config picker returned None for kernel '{self.op_name}' "
|
||||
f"with available config keys: {list(self.configs.keys())}"
|
||||
f"Config picker returned None for kernel "
|
||||
f"'{self.op_name}' with available config keys: "
|
||||
f"{list(self.configs.keys())}"
|
||||
)
|
||||
|
||||
if selected_config_key not in self.configs:
|
||||
config_key = str_to_key.get(selected_str)
|
||||
if config_key is None:
|
||||
raise ValueError(
|
||||
f"Config picker returned invalid config key "
|
||||
f"'{selected_config_key}' for kernel '{self.op_name}'. "
|
||||
f"'{selected_str}' for kernel "
|
||||
f"'{self.op_name}'. "
|
||||
f"Available keys: {list(self.configs.keys())}"
|
||||
)
|
||||
|
||||
return self.configs[selected_config_key]
|
||||
return self.configs[config_key]
|
||||
|
||||
return config_selector
|
||||
|
||||
@@ -251,9 +259,9 @@ class HelionKernelWrapper:
|
||||
raw_kernel_func: Callable,
|
||||
op_name: str,
|
||||
fake_impl: Callable,
|
||||
config_picker: Callable[[tuple[Any, ...], list[str]], str | None],
|
||||
helion_settings: "helion.Settings | None" = None,
|
||||
input_generator: Callable[[], dict[str, tuple[Any, ...]]] | None = None,
|
||||
config_picker: ConfigPicker,
|
||||
helion_settings: helion.Settings | None = None,
|
||||
input_generator: (Callable[[], dict[CaseKey, tuple[Any, ...]]] | None) = None,
|
||||
):
|
||||
# Validate helion_settings doesn't conflict with our custom autotuner
|
||||
validate_helion_settings(helion_settings, op_name)
|
||||
@@ -302,7 +310,7 @@ class HelionKernelWrapper:
|
||||
# During eager execution, call the kernel directly.
|
||||
return self._configured_kernel(*args, **kwargs)
|
||||
|
||||
def get_inputs(self) -> dict[str, tuple[Any, ...]]:
|
||||
def get_inputs(self) -> dict[CaseKey, tuple[Any, ...]]:
|
||||
if self._input_generator is None:
|
||||
raise NotImplementedError(
|
||||
f"No input generator registered for kernel '{self.op_name}'. "
|
||||
@@ -370,7 +378,7 @@ def get_kernel_by_name(kernel_name: str) -> HelionKernelWrapper | None:
|
||||
|
||||
def infer_fake_impl(
|
||||
kernel_func: Callable,
|
||||
helion_settings: "helion.Settings | None" = None,
|
||||
helion_settings: helion.Settings | None = None,
|
||||
) -> Callable:
|
||||
def helion_fake_kernel(*args, **kwargs):
|
||||
kernel_kwargs = {}
|
||||
@@ -392,37 +400,29 @@ def infer_fake_impl(
|
||||
def register_kernel(
|
||||
op_name: str | None = None,
|
||||
*,
|
||||
config_picker: Callable[[tuple[Any, ...], list[str]], str | None],
|
||||
config_picker: ConfigPicker,
|
||||
fake_impl: Callable | None = None,
|
||||
helion_settings: "helion.Settings | None" = None,
|
||||
input_generator: Callable[[], dict[str, tuple[Any, ...]]] | None = None,
|
||||
helion_settings: helion.Settings | None = None,
|
||||
input_generator: (Callable[[], dict[CaseKey, tuple[Any, ...]]] | None) = None,
|
||||
) -> Callable[[Callable], HelionKernelWrapper]:
|
||||
"""Register a Helion kernel with pre-tuned config selection.
|
||||
|
||||
Wraps the kernel function in a HelionKernelWrapper that eagerly builds
|
||||
the configured kernel and (on older PyTorch) registers a custom op.
|
||||
|
||||
Args:
|
||||
config_picker: Required. Function with signature
|
||||
``(args: tuple, config_keys: list[str]) -> str | None``
|
||||
that picks the best config key from available options.
|
||||
Return ``None`` to fall back to ``"default"``.
|
||||
config_picker: Required. Receives ``(args, config_keys)``
|
||||
where each config key is a ``dict[str, Any]`` mapping
|
||||
parameter names to values. Return the best-matching
|
||||
dict, or ``None`` to fall back to the default config.
|
||||
|
||||
Example::
|
||||
|
||||
def pick_config(args, config_keys):
|
||||
x = args[0]
|
||||
hidden_size = x.shape[-1]
|
||||
batch_size = x.shape[0]
|
||||
for key in config_keys:
|
||||
if key == f"hiddensize_{hidden_size}_batchsize_{batch_size}":
|
||||
return key
|
||||
return "default" if "default" in config_keys else None
|
||||
best = min(config_keys, key=lambda k: abs(k["size"] - x.shape[0]))
|
||||
return best
|
||||
|
||||
input_generator: Optional. Function that returns
|
||||
``dict[str, tuple]`` where each key is a configuration
|
||||
identifier (e.g. ``"4096"``, ``"hidden_4096"``) and each
|
||||
value is a tuple of arguments to pass to the kernel.
|
||||
input_generator: Optional. Returns ``dict[str, tuple]`` where
|
||||
each key is a serialized config key and each value is a
|
||||
tuple of arguments to pass to the kernel.
|
||||
|
||||
Example::
|
||||
|
||||
|
||||
Reference in New Issue
Block a user