[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
-75
View File
@@ -1,75 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import tempfile
from collections.abc import Callable
from contextlib import contextmanager
from pathlib import Path
from typing import Any
from unittest.mock import patch
import helion
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.register import register_kernel
from vllm.kernels.helion.utils import get_canonical_gpu_name
GPU_PLATFORM = get_canonical_gpu_name()
DEFAULT_CONFIGS: dict[CaseKey, helion.Config] = {
CaseKey.default(): helion.Config(block_sizes=[32]),
}
@contextmanager
def dummy_kernel_registry(
configs: dict[CaseKey, helion.Config] | None = None,
):
"""Context manager providing a register function with automatic config setup.
Yields a ``register`` callable with the same signature as
``register_kernel``. Before applying the real decorator it writes a
config JSON for the kernel name (from ``op_name`` or ``fn.__name__``)
into a temporary directory backed by a fresh ``ConfigManager``.
"""
if configs is None:
configs = DEFAULT_CONFIGS
def _to_config_entries(cfgs: dict) -> list[dict[str, Any]]:
pairs: list[dict[str, Any]] = []
for k, v in cfgs.items():
config_data = v.__dict__["config"]
pairs.append({"key": dict(k), "config": config_data})
return pairs
with tempfile.TemporaryDirectory() as tmpdir:
config_dir = Path(tmpdir)
ConfigManager.reset_instance()
cm = ConfigManager(base_dir=config_dir)
with patch(
"vllm.kernels.helion.config_manager.ConfigManager",
return_value=cm,
):
def register(
op_name: str | None = None,
**kwargs,
) -> Callable:
def decorator(fn: Callable) -> Callable:
name = op_name or fn.__name__
kernel_dir = config_dir / name
kernel_dir.mkdir(parents=True, exist_ok=True)
(kernel_dir / f"{GPU_PLATFORM}.json").write_text(
json.dumps(_to_config_entries(configs))
)
return register_kernel(op_name, **kwargs)(fn)
return decorator
try:
yield register
finally:
ConfigManager.reset_instance()
-91
View File
@@ -1,91 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for autotuning Helion kernels, including disabled kernels with no configs."""
import pytest
import torch
from vllm.utils.import_utils import has_helion
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
import helion
import helion.language as hl
from helion.autotuner.base_search import BaseSearch
from tests.kernels.helion.helpers import dummy_kernel_registry
from vllm.kernels.helion.register import create_helion_decorated_kernel
def _add_kernel(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
out = torch.empty_like(x)
for tile in hl.tile(x.size()):
out[tile] = x[tile] + y[tile]
return out
class NoCompileSearch(BaseSearch):
"""Autotuner that returns the default config without GPU compilation.
Modeled after helion's test BasicSearch (pytorch/helion#1649).
"""
def autotune(self, *, skip_cache: bool = False):
return self.config_spec.default_config()
def _no_compile_autotuner_fn(bound_kernel, args, **kwargs):
return NoCompileSearch(bound_kernel, args, **kwargs)
class TestAutotuneDisabledKernel:
"""Test autotuning flow on disabled kernels (no platform configs)."""
def setup_method(self):
from vllm.kernels.helion.register import _REGISTERED_KERNELS
self._saved_registry = dict(_REGISTERED_KERNELS)
_REGISTERED_KERNELS.clear()
def teardown_method(self):
from vllm.kernels.helion.register import _REGISTERED_KERNELS
_REGISTERED_KERNELS.clear()
_REGISTERED_KERNELS.update(self._saved_registry)
def test_autotune_disabled_kernel_produces_valid_config(self):
"""Register a kernel with no configs (disabled), run autotune,
verify it produces a valid helion.Config."""
with dummy_kernel_registry(configs={}) as register:
wrapper = register(
"autotune_test_kernel",
config_picker=lambda args, keys: None,
fake_impl=lambda *a, **kw: None,
input_generator=lambda: {
"small": (
torch.randn(4, 4, device="cuda"),
torch.randn(4, 4, device="cuda"),
),
},
)(_add_kernel)
assert wrapper._disabled is True
inputs = wrapper.get_inputs()
assert "small" in inputs
settings = helion.Settings()
settings.autotuner_fn = _no_compile_autotuner_fn
wrapper.helion_settings = settings
config = wrapper.run_autotune(inputs["small"])
expected_default = (
create_helion_decorated_kernel(_add_kernel, helion_settings=settings)
.bind(inputs["small"])
.config_spec.default_config()
)
assert config == expected_default
-68
View File
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.utils.import_utils import has_helion
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
from vllm.kernels.helion.case_key import CaseKey
class TestCaseKey:
"""Test suite for CaseKey class."""
def test_construction_with_dict(self):
key = CaseKey({"intermediate": 2048, "numtokens": 256})
assert key["intermediate"] == 2048
assert key["numtokens"] == 256
def test_empty_construction_raises(self):
with pytest.raises(TypeError, match="at least one key-value pair"):
CaseKey()
with pytest.raises(TypeError, match="at least one key-value pair"):
CaseKey({})
def test_default_construction(self):
key = CaseKey.default()
assert len(key) == 0
assert key.is_default()
def test_non_default_is_not_default(self):
key = CaseKey({"intermediate": 2048})
assert not key.is_default()
def test_hashable_and_equality(self):
a = CaseKey({"intermediate": 2048, "numtokens": 256})
b = CaseKey({"numtokens": 256, "intermediate": 2048})
assert a == b
assert hash(a) == hash(b)
assert a != CaseKey({"intermediate": 4096})
assert CaseKey.default() == CaseKey.default()
configs = {
CaseKey.default(): "default_config",
a: "a_config",
}
assert configs[b] == "a_config"
assert configs[CaseKey.default()] == "default_config"
def test_str_is_sorted_json(self):
assert str(CaseKey({"z": 1, "a": 2})) == '{"a":2,"z":1}'
assert str(CaseKey.default()) == "{}"
def test_immutable(self):
key = CaseKey({"intermediate": 2048})
with pytest.raises(TypeError, match="immutable"):
key["intermediate"] = 4096
with pytest.raises(TypeError, match="immutable"):
del key["intermediate"]
with pytest.raises(TypeError, match="immutable"):
key.update({"numtokens": 256})
with pytest.raises(TypeError, match="immutable"):
key.clear()
-408
View File
@@ -1,408 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for Helion ConfigManager and ConfigSet.
Tests the simplified configuration management system for Helion custom kernels.
"""
import json
import tempfile
from pathlib import Path
import pytest
from vllm.utils.import_utils import has_helion
# Skip entire module if helion is not available
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
import helion
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import (
ConfigManager,
ConfigSet,
)
@pytest.fixture(autouse=True)
def reset_config_manager_singleton():
"""Reset ConfigManager singleton before each test."""
ConfigManager.reset_instance()
yield
ConfigManager.reset_instance()
class TestConfigSet:
"""Test suite for ConfigSet class."""
def test_config_set_creation(self):
"""Test creating an empty ConfigSet."""
config_set = ConfigSet("test_kernel")
assert config_set.kernel_name == "test_kernel"
assert config_set.get_platforms() == []
def test_config_set_from_dict(self):
"""Test creating ConfigSet from dictionary data."""
config_data = {
"block_sizes": [32, 16],
"num_warps": 4,
"num_stages": 3,
"pid_type": "persistent_interleaved",
}
data = {
"h100": [
{"key": {"batch": 32, "hidden": 4096}, "config": config_data},
]
}
config_set = ConfigSet.from_dict("test_kernel", data)
assert config_set.kernel_name == "test_kernel"
assert config_set.get_platforms() == ["h100"]
internal_key = CaseKey({"batch": 32, "hidden": 4096})
config = config_set.get_config("h100", internal_key)
assert isinstance(config, helion.Config)
assert config.block_sizes == [32, 16]
assert config.num_warps == 4
assert config.num_stages == 3
assert config.pid_type == "persistent_interleaved"
def test_config_set_get_config_keyerror(self):
"""Test that accessing non-existent configs raises informative KeyErrors."""
config_set = ConfigSet("test_kernel")
with pytest.raises(KeyError, match="platform 'h100' not found"):
config_set.get_config("h100", CaseKey({"batch": 32}))
config_data = {"num_warps": 8, "num_stages": 4}
data = {
"h100": [
{"key": {"batch": 64, "hidden": 2048}, "config": config_data},
]
}
config_set = ConfigSet.from_dict("test_kernel", data)
nonexistent_key = CaseKey({"batch": 32, "hidden": 4096})
with pytest.raises(KeyError, match="config_key .* not found"):
config_set.get_config("h100", nonexistent_key)
def test_config_set_get_platforms(self):
"""Test get_platforms method."""
# Use realistic config data
config1 = {"num_warps": 4, "num_stages": 3}
config2 = {"num_warps": 8, "num_stages": 5}
data = {
"h100": [
{"key": {"batch": 32, "hidden": 4096}, "config": config1},
],
"a100": [
{"key": {"batch": 16, "hidden": 2048}, "config": config2},
],
}
config_set = ConfigSet.from_dict("test_kernel", data)
platforms = config_set.get_platforms()
assert platforms == ["a100", "h100"] # Should be sorted
def test_config_set_get_config_keys(self):
"""Test get_config_keys method."""
config1 = {"num_warps": 4, "num_stages": 3}
config2 = {"num_warps": 8, "num_stages": 5}
data = {
"h100": [
{"key": {"batch": 32, "hidden": 4096}, "config": config1},
{"key": {"batch": 64, "hidden": 2048}, "config": config2},
]
}
config_set = ConfigSet.from_dict("test_kernel", data)
config_keys = config_set.get_config_keys("h100")
expected_keys = sorted(
[
CaseKey({"batch": 32, "hidden": 4096}),
CaseKey({"batch": 64, "hidden": 2048}),
],
key=lambda k: str(k) if k is not None else "",
)
assert config_keys == expected_keys
assert config_set.get_config_keys("v100") == []
def test_config_set_to_dict(self):
"""Test converting ConfigSet to dictionary."""
original_config = {
"block_sizes": [64, 32],
"num_warps": 16,
"num_stages": 4,
"pid_type": "persistent_blocked",
}
original_data = {
"h100": [
{"key": {"batch": 32, "hidden": 4096}, "config": original_config},
]
}
config_set = ConfigSet.from_dict("test_kernel", original_data)
result_data = config_set.to_dict()
internal_key = CaseKey({"batch": 32, "hidden": 4096})
assert internal_key in result_data["h100"]
assert result_data["h100"][internal_key] == original_config
class TestConfigManager:
"""Test suite for ConfigManager class."""
def test_config_manager_creation_default_base_dir(self):
"""Test creating ConfigManager with default base directory."""
manager = ConfigManager()
assert manager._base_dir.name == "configs"
def test_config_manager_creation_custom_base_dir(self):
"""Test creating ConfigManager with custom base directory."""
custom_dir = "/tmp/custom_configs"
manager = ConfigManager(base_dir=custom_dir)
# Paths are resolved, so compare with resolved path
assert manager._base_dir == Path(custom_dir).resolve()
def test_get_config_file_path(self):
"""Test getting config file path for a kernel."""
manager = ConfigManager(base_dir="/tmp")
dir_path = manager.get_config_file_path("silu_mul_fp8")
assert dir_path == Path("/tmp/silu_mul_fp8")
file_path = manager.get_config_file_path("silu_mul_fp8", "nvidia_h100")
assert file_path == Path("/tmp/silu_mul_fp8/nvidia_h100.json")
def test_ensure_base_dir_exists(self):
"""Test ensuring base directory exists."""
with tempfile.TemporaryDirectory() as temp_dir:
base_dir = Path(temp_dir) / "non_existent" / "configs"
manager = ConfigManager(base_dir=base_dir)
assert not base_dir.exists()
returned_path = manager.ensure_base_dir_exists()
assert base_dir.exists()
assert base_dir.is_dir()
assert returned_path == base_dir
def test_load_config_set_file_not_exists(self):
"""Test loading config set when file doesn't exist."""
with tempfile.TemporaryDirectory() as temp_dir:
manager = ConfigManager(base_dir=temp_dir)
config_set = manager.load_config_set("non_existent_kernel")
assert isinstance(config_set, ConfigSet)
assert config_set.kernel_name == "non_existent_kernel"
assert config_set.get_platforms() == []
def test_load_config_set_valid_file(self):
"""Test loading config set from per-platform files."""
with tempfile.TemporaryDirectory() as temp_dir:
kernel_config = {
"block_sizes": [128, 64],
"num_warps": 8,
"num_stages": 6,
"pid_type": "persistent_interleaved",
}
kernel_dir = Path(temp_dir) / "test_kernel"
kernel_dir.mkdir()
platform_file = kernel_dir / "h100.json"
with open(platform_file, "w") as f:
json.dump(
[{"key": {"batch": 32, "hidden": 4096}, "config": kernel_config}],
f,
)
manager = ConfigManager(base_dir=temp_dir)
config_set = manager.load_config_set("test_kernel")
assert isinstance(config_set, ConfigSet)
assert config_set.kernel_name == "test_kernel"
assert config_set.get_platforms() == ["h100"]
internal_key = CaseKey({"batch": 32, "hidden": 4096})
config = config_set.get_config("h100", internal_key)
assert isinstance(config, helion.Config)
assert config.block_sizes == [128, 64]
assert config.num_warps == 8
def test_load_config_set_invalid_json(self):
"""Test loading config set from file with invalid JSON."""
with tempfile.TemporaryDirectory() as temp_dir:
kernel_dir = Path(temp_dir) / "test_kernel"
kernel_dir.mkdir()
config_file = kernel_dir / "h100.json"
with open(config_file, "w") as f:
f.write("invalid json content {")
manager = ConfigManager(base_dir=temp_dir)
config_set = manager.load_config_set("test_kernel")
assert isinstance(config_set, ConfigSet)
assert config_set.kernel_name == "test_kernel"
assert config_set.get_platforms() == []
def test_save_config_set(self):
"""Test saving ConfigSet to per-platform files."""
with tempfile.TemporaryDirectory() as temp_dir:
kernel_config = {
"block_sizes": [256, 128],
"num_warps": 16,
"num_stages": 8,
"pid_type": "persistent_blocked",
}
data = {
"h100": [
{"key": {"batch": 32, "hidden": 4096}, "config": kernel_config},
]
}
config_set = ConfigSet.from_dict("test_kernel", data)
manager = ConfigManager(base_dir=temp_dir)
saved_path = manager.save_config_set(config_set)
expected_dir = Path(temp_dir) / "test_kernel"
assert saved_path == expected_dir
assert saved_path.is_dir()
platform_file = expected_dir / "h100.json"
assert platform_file.exists()
with open(platform_file) as f:
loaded_data = json.load(f)
assert isinstance(loaded_data, list)
assert len(loaded_data) == 1
entry = loaded_data[0]
assert entry["key"] == {"batch": 32, "hidden": 4096}
assert entry["config"] == kernel_config
def test_save_config_set_creates_directory(self):
"""Test that save_config_set creates parent directories if needed."""
with tempfile.TemporaryDirectory() as temp_dir:
nested_dir = Path(temp_dir) / "nested" / "configs"
data = {
"h100": [
{"key": {}, "config": {"num_warps": 4}},
]
}
config_set = ConfigSet.from_dict("test_kernel", data)
manager = ConfigManager(base_dir=nested_dir)
saved_path = manager.save_config_set(config_set)
assert nested_dir.exists()
assert nested_dir.is_dir()
assert saved_path.is_dir()
assert (saved_path / "h100.json").exists()
def test_get_platform_configs(self):
"""Test getting all configs for a specific platform."""
with tempfile.TemporaryDirectory() as temp_dir:
config_1 = {"num_warps": 4, "num_stages": 3, "block_sizes": [64, 32]}
config_2 = {"num_warps": 8, "num_stages": 5, "block_sizes": [128, 64]}
default_config = {
"num_warps": 16,
"num_stages": 7,
"block_sizes": [256, 128],
}
config_3 = {"num_warps": 2, "num_stages": 2, "block_sizes": [32, 16]}
kernel_dir = Path(temp_dir) / "test_kernel"
kernel_dir.mkdir()
with open(kernel_dir / "h100.json", "w") as f:
json.dump(
[
{"key": {"batch": 32, "hidden": 4096}, "config": config_1},
{"key": {"batch": 64, "hidden": 2048}, "config": config_2},
{"key": {}, "config": default_config},
],
f,
)
with open(kernel_dir / "a100.json", "w") as f:
json.dump(
[{"key": {"batch": 16, "hidden": 1024}, "config": config_3}],
f,
)
manager = ConfigManager(base_dir=temp_dir)
key_b32_h4096 = CaseKey({"batch": 32, "hidden": 4096})
key_b64_h2048 = CaseKey({"batch": 64, "hidden": 2048})
key_b16_h1024 = CaseKey({"batch": 16, "hidden": 1024})
h100_configs = manager.get_platform_configs("test_kernel", "h100")
assert len(h100_configs) == 3
assert key_b32_h4096 in h100_configs
assert key_b64_h2048 in h100_configs
assert CaseKey.default() in h100_configs
for config in h100_configs.values():
assert isinstance(config, helion.Config)
assert h100_configs[key_b32_h4096].num_warps == 4
assert h100_configs[CaseKey.default()].num_stages == 7
a100_configs = manager.get_platform_configs("test_kernel", "a100")
assert len(a100_configs) == 1
assert key_b16_h1024 in a100_configs
assert isinstance(a100_configs[key_b16_h1024], helion.Config)
assert a100_configs[key_b16_h1024].num_warps == 2
nonexistent_configs = manager.get_platform_configs("test_kernel", "v100")
assert len(nonexistent_configs) == 0
def test_singleton_returns_same_instance(self):
"""Test that ConfigManager returns the same instance on repeated calls."""
manager1 = ConfigManager(base_dir="/tmp/test_singleton")
manager2 = ConfigManager(base_dir="/tmp/test_singleton")
assert manager1 is manager2
def test_singleton_with_default_base_dir(self):
"""Test singleton behavior with default base directory."""
manager1 = ConfigManager()
manager2 = ConfigManager()
assert manager1 is manager2
assert manager1._base_dir == manager2._base_dir
def test_singleton_error_on_different_base_dir(self):
"""Test that ConfigManager raises error when created with different base_dir."""
ConfigManager(base_dir="/tmp/first_dir")
with pytest.raises(ValueError, match="singleton already exists"):
ConfigManager(base_dir="/tmp/different_dir")
def test_reset_instance_allows_new_base_dir(self):
"""Test that reset_instance allows creating with a new base_dir."""
manager1 = ConfigManager(base_dir="/tmp/first_dir")
assert manager1._base_dir == Path("/tmp/first_dir").resolve()
ConfigManager.reset_instance()
manager2 = ConfigManager(base_dir="/tmp/second_dir")
assert manager2._base_dir == Path("/tmp/second_dir").resolve()
assert manager1 is not manager2
def test_get_instance_returns_existing(self):
"""Test that get_instance returns the existing singleton."""
manager1 = ConfigManager(base_dir="/tmp/test_get_instance")
manager2 = ConfigManager.get_instance()
assert manager1 is manager2
def test_get_instance_raises_if_not_initialized(self):
"""Test that get_instance raises RuntimeError if no instance exists."""
with pytest.raises(RuntimeError, match="has not been created"):
ConfigManager.get_instance()
@@ -1,154 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the dynamic_per_token_scaled_fp8_quant helion kernel.
Run `pytest tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py`.
"""
from typing import Any
import pytest
import torch
from torch._subclasses.fake_tensor import FakeTensorMode
from tests.kernels.helion.utils import skip_if_platform_unsupported
from tests.kernels.quant_utils import FP8_DTYPE
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.ops.dynamic_per_token_scaled_fp8_quant import (
_pick_cache,
baseline,
dynamic_per_token_scaled_fp8_quant,
pick_config,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
from vllm.utils.torch_utils import set_random_seed
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
def _generate_fake_input(num_tokens: int, hidden_size: int) -> tuple[Any, ...]:
with FakeTensorMode():
input = torch.randn(
num_tokens, hidden_size, device="cuda", dtype=torch.bfloat16
)
result = torch.empty(
input.shape, device=input.device, dtype=current_platform.fp8_dtype()
)
scale = torch.empty((num_tokens, 1), device=input.device, dtype=torch.float32)
scale_ub = torch.mean(input).to(torch.float32)
args = (result, input, scale, scale_ub)
return args
@pytest.fixture(autouse=True)
def reset_config_manager_singleton():
ConfigManager.reset_instance()
ConfigManager()
yield
ConfigManager.reset_instance()
class TestDynamicPerTokenScaledFp8QuantConfigPicker:
def setup_method(self):
_pick_cache.clear()
def test_config_picker_exact_match(self):
config_keys = [
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
]
args = _generate_fake_input(16, 4096)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey({"hidden_size": 4096, "num_tokens": 16})
def test_config_picker_closest_match(self):
config_keys = [
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
CaseKey({"hidden_size": 2048, "num_tokens": 32}),
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "num_tokens": 32}),
]
args = _generate_fake_input(20, 3000)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
def test_config_picker_no_configs(self):
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096)
selected_key = pick_config(args, config_keys)
assert selected_key is None
def test_config_picker_fallback_to_largest(self):
config_keys = [
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
]
args = _generate_fake_input(32, 8192)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey({"hidden_size": 4096, "num_tokens": 16})
class TestDynamicPerTokenScaledFp8QuantCorrectness:
@pytest.mark.parametrize("num_tokens", [1, 7, 4096])
@pytest.mark.parametrize("hidden_size", [17, 1024, 1025, 1026, 5137, 8193])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float])
@pytest.mark.parametrize("has_scale_ub", [True, False])
@pytest.mark.parametrize("seed", [0])
def test_dynamic_per_token_fp8_quant(
self,
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
has_scale_ub: bool,
seed: int,
) -> None:
skip_if_platform_unsupported("dynamic_per_token_scaled_fp8_quant")
set_random_seed(seed)
x = (
torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") + 1e-6
) # avoid nans
scale_ub = (
torch.mean(x).to(dtype=torch.float32, device="cuda")
if has_scale_ub
else None
)
ref_out = torch.empty(x.shape, device="cuda", dtype=FP8_DTYPE)
ref_scales = torch.empty((x.shape[0], 1), device="cuda", dtype=torch.float32)
baseline(ref_out, x, ref_scales, scale_ub)
ops_out = torch.empty(x.shape, device="cuda", dtype=FP8_DTYPE)
ops_scales = torch.empty((x.shape[0], 1), device="cuda", dtype=torch.float32)
dynamic_per_token_scaled_fp8_quant(ops_out, x, ops_scales, scale_ub)
torch.testing.assert_close(ref_scales, ops_scales)
# allow 1 ULP difference
assert (
ref_out.view(torch.uint8).to(torch.int16)
- ops_out.view(torch.uint8).to(torch.int16)
).abs().max() <= 1
class TestDynamicPerTokenScaledFp8QuantIntegration:
def test_kernel_registration_integration(self):
from vllm.kernels.helion.register import get_registered_kernels
registered_kernels = get_registered_kernels()
assert "dynamic_per_token_scaled_fp8_quant" in registered_kernels
kernel_wrapper = registered_kernels["dynamic_per_token_scaled_fp8_quant"]
assert kernel_wrapper.op_name == "dynamic_per_token_scaled_fp8_quant"
assert kernel_wrapper._config_picker is not None
assert kernel_wrapper._mutates_args == ["result", "scale"]
@@ -1,206 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test make_fx tracing and inductor pattern matching with HelionKernelWrapper."""
import contextlib
from unittest.mock import Mock, patch
import pytest
import torch
from vllm.utils.import_utils import has_helion
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
import helion
import helion.language as hl
from helion._compat import requires_torch_version
if not requires_torch_version("2.11"):
pytest.skip(
"HigherOrderOp requires PyTorch >= 2.11",
allow_module_level=True,
)
from helion._compiler._dynamo.higher_order_ops import (
helion_kernel_side_table,
helion_kernel_wrapper_mutation,
)
from torch._inductor.pattern_matcher import (
PatternMatcherPass,
fwd_only,
register_replacement,
select_decomp_table,
)
from torch.fx.experimental.proxy_tensor import make_fx
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.register import HelionKernelWrapper
@contextlib.contextmanager
def _helion_mock_context():
configs = {
CaseKey.default(): helion.Config(block_sizes=[64], num_warps=2, num_stages=2),
}
mock_config_manager = Mock(spec=ConfigManager)
mock_config_manager.get_platform_configs = Mock(return_value=configs)
with (
patch(
"vllm.kernels.helion.config_manager.ConfigManager",
return_value=mock_config_manager,
),
patch(
"vllm.kernels.helion.utils.get_canonical_gpu_name",
return_value="nvidia_h200",
),
):
yield
class TestMakeFxHop:
def setup_method(self):
helion_kernel_side_table.reset_table()
@pytest.mark.skip(reason="SymInt proxy tracking issue with PyTorch 2.11+")
def test_make_fx_symbolic(self):
def raw_add_scale(
x: torch.Tensor, y: torch.Tensor, scale: float
) -> tuple[torch.Tensor, int, torch.Tensor]:
out_x = torch.empty_like(x)
out_y = torch.empty_like(x)
for tile in hl.tile(x.size()):
out_x[tile] = x[tile] + y[tile] * scale
out_y[tile] = out_x[tile] * 2.0
return out_x, 42, out_y
input_x = torch.randn(7, 13)
input_y = torch.randn(7, 13)
scale = 0.5
with _helion_mock_context():
wrapper = HelionKernelWrapper(
raw_kernel_func=raw_add_scale,
op_name="test_make_fx",
fake_impl=lambda *a, **kw: None,
config_picker=lambda args, keys: CaseKey.default(),
)
def fn(x, y):
return wrapper(x, y, scale)
gm = make_fx(fn, tracing_mode="symbolic")(input_x, input_y)
hop_nodes = [
n
for n in gm.graph.nodes
if n.op == "call_function" and n.target is helion_kernel_wrapper_mutation
]
assert len(hop_nodes) == 1
node = hop_nodes[0]
assert node.kwargs["constant_args"]["scale"] == scale
assert set(node.kwargs["tensor_args"]) == {"x", "y"}
specs = node.kwargs["output_spec"]["leaf_specs"]
tensor_specs = [s for s in specs if s["type"] == "tensor"]
scalar_specs = [s for s in specs if s["type"] == "scalar"]
assert len(tensor_specs) == 2
assert len(scalar_specs) == 1
for spec in tensor_specs:
assert spec["dtype"] == input_x.dtype
assert scalar_specs[0]["scalar_value"] == 42
for val in node.meta["val"]:
assert all(isinstance(s, torch.SymInt) for s in val.shape)
# Both out_x and out_y are empty_like(x), so output shapes == input shape
input_node = next(n for n in gm.graph.nodes if n.op == "placeholder")
input_shape = input_node.meta["val"].shape
for val in node.meta["val"]:
assert len(val.shape) == len(input_shape)
for out_s, in_s in zip(val.shape, input_shape):
assert out_s == in_s
@pytest.mark.skip(reason="SymInt proxy tracking issue with PyTorch 2.11+")
def test_pattern_matcher_replaces_with_helion_hop(self):
def raw_silu_mul(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
M, N = x.size()
out = torch.empty_like(x)
for tile_m, tile_n in hl.tile([M, N]):
out[tile_m, tile_n] = (
torch.nn.functional.silu(x[tile_m, tile_n]) * y[tile_m, tile_n]
)
return out
with _helion_mock_context():
wrapper = HelionKernelWrapper(
raw_kernel_func=raw_silu_mul,
op_name="test_pm_silu_mul",
fake_impl=lambda *a, **kw: None,
config_picker=lambda args, keys: CaseKey.default(),
)
def pattern(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return torch.nn.functional.silu(x) * y
def replacement(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return wrapper(x, y)
inputs = [torch.randn(8, 16), torch.randn(8, 16)]
pm_pass = PatternMatcherPass(pass_name="test_helion_replacement")
register_replacement(pattern, replacement, inputs, fwd_only, pm_pass)
def model(x, y):
return torch.nn.functional.silu(x) * y
decompositions = select_decomp_table()
input_x = torch.randn(8, 16)
input_y = torch.randn(8, 16)
gm = make_fx(model, decompositions, tracing_mode="symbolic")(
input_x, input_y
)
def count_hop_nodes(graph):
return sum(
1
for n in graph.nodes
if n.op == "call_function"
and n.target is helion_kernel_wrapper_mutation
)
assert count_hop_nodes(gm.graph) == 0
match_count = pm_pass.apply(gm.graph)
gm.graph.lint()
gm.recompile()
assert match_count == 1
assert count_hop_nodes(gm.graph) == 1
hop_node = next(
n
for n in gm.graph.nodes
if n.op == "call_function"
and n.target is helion_kernel_wrapper_mutation
)
# raw_silu_mul returns empty_like(x), so output shape == input shape
for val in hop_node.meta["val"]:
assert all(isinstance(s, torch.SymInt) for s in val.shape)
input_node = next(n for n in gm.graph.nodes if n.op == "placeholder")
input_shape = input_node.meta["val"].shape
output_shape = hop_node.meta["val"][0].shape
assert len(output_shape) == len(input_shape)
for out_s, in_s in zip(output_shape, input_shape):
assert out_s == in_s
@@ -1,232 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the per_token_group_fp8_quant helion kernel.
Run `pytest tests/kernels/helion/test_per_token_group_fp8_quant.py`.
"""
from typing import Any
import pytest
import torch
from torch._subclasses.fake_tensor import FakeTensorMode
from tests.kernels.helion.utils import skip_if_platform_unsupported
from tests.kernels.quant_utils import FP8_DTYPE
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.ops.per_token_group_fp8_quant import (
_pick_cache,
baseline,
per_token_group_fp8_quant,
pick_config,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.utils.import_utils import has_helion
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
def _generate_fake_input(
num_tokens: int, hidden_size: int, group_size: int
) -> tuple[Any, ...]:
with FakeTensorMode():
input = torch.randn(
(num_tokens, hidden_size), device="cuda", dtype=torch.bfloat16
)
output_q = torch.empty(input.shape, device=input.device, dtype=FP8_DTYPE)
output_s = torch.empty(
(num_tokens, hidden_size // group_size),
device=input.device,
dtype=torch.float32,
)
use_ue8m0 = False
column_major = False
fp8_min, fp8_max = get_fp8_min_max()
eps = 1e-10
args = (
input,
output_q,
output_s,
group_size,
eps,
fp8_min,
fp8_max,
use_ue8m0,
column_major,
)
return args
@pytest.fixture(autouse=True)
def reset_config_manager_singleton():
ConfigManager.reset_instance()
ConfigManager()
yield
ConfigManager.reset_instance()
class TestPerTokenGroupFp8QuantConfigPicker:
def setup_method(self):
_pick_cache.clear()
def test_config_picker_exact_match(self):
config_keys = [
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
]
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"hidden_size": 4096, "group_size": 128, "num_tokens": 16}
)
def test_config_picker_closest_match(self):
config_keys = [
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 32}),
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 16}),
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 32}),
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 32}),
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 32}),
]
args = _generate_fake_input(20, 3000, 70)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"hidden_size": 2048, "group_size": 64, "num_tokens": 32}
)
def test_config_picker_no_configs(self):
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
assert selected_key is None
def test_config_picker_fallback_to_largest(self):
config_keys = [
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 32}),
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 16}),
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 32}),
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 32}),
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 32}),
]
args = _generate_fake_input(64, 8192, 256)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"hidden_size": 4096, "group_size": 128, "num_tokens": 32}
)
class TestPerTokenGroupFp8QuantCorrectness:
@pytest.mark.parametrize(
"shape", [(31, 128), (32, 128), (63, 256), (64, 256), (16, 512), (2048, 5120)]
)
@pytest.mark.parametrize("column_major", [False, True])
@pytest.mark.parametrize("tma_aligned", [False, True])
@pytest.mark.parametrize("scale_ue8m0", [False, True])
@pytest.mark.parametrize("group_size", [64, 128])
def test_per_token_group_fp8_quant(
self,
shape,
column_major: bool,
tma_aligned: bool,
scale_ue8m0: bool,
group_size: int,
):
skip_if_platform_unsupported("per_token_group_fp8_quant")
torch.manual_seed(42)
num_tokens, hidden_size = shape
fp8_min, fp8_max = get_fp8_min_max()
eps = 1e-10
input = (
torch.randn((num_tokens, hidden_size), device="cuda", dtype=torch.bfloat16)
* 8
)
ref_q = torch.empty(input.shape, device=input.device, dtype=FP8_DTYPE)
ops_q = ref_q.clone()
groups_per_row = hidden_size // group_size
if column_major:
if tma_aligned:
tma_alignment = 4
tma_aligned_m = (
(num_tokens + tma_alignment - 1) // tma_alignment * tma_alignment
)
shape = (num_tokens, groups_per_row)
stride = (1, tma_aligned_m)
ref_s = torch.empty_strided(
shape, stride, device=input.device, dtype=torch.float32
)
else:
ref_s = torch.empty(
(groups_per_row, num_tokens),
device=input.device,
dtype=torch.float32,
).transpose(0, 1)
else:
ref_s = torch.empty(
(num_tokens, groups_per_row), device=input.device, dtype=torch.float32
)
ops_s = ref_s.clone()
baseline(
input,
ref_q,
ref_s,
group_size,
eps,
fp8_min,
fp8_max,
scale_ue8m0,
column_major,
tma_aligned,
)
per_token_group_fp8_quant(
input,
ops_q,
ops_s,
group_size,
eps,
fp8_min,
fp8_max,
scale_ue8m0,
column_major,
tma_aligned,
)
assert torch.allclose(ref_s, ops_s)
# allow 1 ULP difference
assert (
ref_q.view(torch.uint8).to(torch.int16)
- ops_q.view(torch.uint8).to(torch.int16)
).abs().max() <= 1
class TestPerTokenGroupFp8QuantIntegration:
def test_kernel_registration_integration(self):
from vllm.kernels.helion.register import get_registered_kernels
registered_kernels = get_registered_kernels()
assert "per_token_group_fp8_quant" in registered_kernels
kernel_wrapper = registered_kernels["per_token_group_fp8_quant"]
assert kernel_wrapper.op_name == "per_token_group_fp8_quant"
assert kernel_wrapper._config_picker is not None
assert kernel_wrapper._mutates_args == ["output_q", "output_s"]
File diff suppressed because it is too large Load Diff
@@ -1,196 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the rms_norm_dynamic_per_token_quant helion kernel.
Run `pytest tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py`.
"""
from typing import Any
import pytest
import torch
from torch._subclasses.fake_tensor import FakeTensorMode
from tests.kernels.helion.utils import skip_if_platform_unsupported
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.ops.rms_norm_dynamic_per_token_quant import (
_pick_cache,
baseline,
pick_config,
rms_norm_dynamic_per_token_quant,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
from vllm.utils.torch_utils import set_random_seed
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
def _generate_fake_input(num_tokens: int, hidden_size: int) -> tuple[Any, ...]:
with FakeTensorMode():
input = torch.randn(
num_tokens, hidden_size, device="cuda", dtype=torch.bfloat16
)
result = torch.empty(
input.shape, device=input.device, dtype=current_platform.fp8_dtype()
)
scale = torch.empty((num_tokens, 1), device=input.device, dtype=torch.float32)
scale_ub = torch.mean(input).to(torch.float32)
residual = torch.randn_like(input)
weight = torch.normal(
mean=1.0,
std=1.0,
size=(hidden_size,),
dtype=input.dtype,
device=input.device,
)
epsilon = 1e-6
args = (result, input, weight, scale, epsilon, scale_ub, residual)
return args
@pytest.fixture(autouse=True)
def reset_config_manager_singleton():
ConfigManager.reset_instance()
ConfigManager()
yield
ConfigManager.reset_instance()
class TestRmsNormDynamicPerTokenQuantConfigPicker:
def setup_method(self):
_pick_cache.clear()
def test_config_picker_exact_match(self):
config_keys = [
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
]
args = _generate_fake_input(16, 4096)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey({"hidden_size": 4096, "num_tokens": 16})
def test_config_picker_closest_match(self):
config_keys = [
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
CaseKey({"hidden_size": 2048, "num_tokens": 32}),
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "num_tokens": 32}),
]
args = _generate_fake_input(20, 3000)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
def test_config_picker_no_configs(self):
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096)
selected_key = pick_config(args, config_keys)
assert selected_key is None
def test_config_picker_fallback_to_largest(self):
config_keys = [
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
]
args = _generate_fake_input(32, 8192)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey({"hidden_size": 4096, "num_tokens": 16})
DTYPES = [torch.bfloat16, torch.float]
QUANT_DTYPES = [torch.int8, current_platform.fp8_dtype()]
VEC_HIDDEN_SIZES = [1024, 1025, 1027, 1029]
# Avoid combinatorial explosion with full Cartesian product
NUM_TOKENS_HIDDEN_SIZES = [
*[(1, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5120, 5137]],
*[(2048, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5137]],
*[(4096, i) for i in [1, 64, 5137]],
]
ADD_RESIDUAL = [False, True]
SCALE_UBS = [True, False]
SEEDS = [0]
EPS = 1e-6
class TestRmsNormDynamicPerTokenQuantCorrectness:
@pytest.mark.parametrize("num_tokens, hidden_size", NUM_TOKENS_HIDDEN_SIZES)
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
@pytest.mark.parametrize("has_scale_ub", SCALE_UBS)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("quant_dtype", QUANT_DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
def test_rms_norm_dynamic_per_token_quant(
self,
num_tokens: int,
hidden_size: int,
add_residual: bool,
has_scale_ub: bool,
dtype: torch.dtype,
quant_dtype: torch.dtype,
seed: int,
) -> None:
skip_if_platform_unsupported("rms_norm_dynamic_per_token_quant")
set_random_seed(seed)
if has_scale_ub and quant_dtype != current_platform.fp8_dtype():
# skip
return
scale = 1 / (hidden_size)
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device="cuda") * scale
weight = torch.normal(
mean=1.0, std=1.0, size=(hidden_size,), dtype=dtype, device=x.device
)
residual = torch.randn_like(x) * scale if add_residual else None
scale_ub = (
torch.mean(x).to(dtype=torch.float32, device="cuda")
if has_scale_ub
else None
)
ref_out = torch.empty(x.shape, device=x.device, dtype=quant_dtype)
ref_scales = torch.empty((x.shape[0], 1), device=x.device, dtype=torch.float32)
ref_residual = residual.clone() if residual is not None else None
baseline(ref_out, x, weight, ref_scales, EPS, scale_ub, ref_residual)
ops_out = torch.empty(x.shape, device=x.device, dtype=quant_dtype)
ops_scales = torch.empty((x.shape[0], 1), device=x.device, dtype=torch.float32)
ops_residual = residual.clone() if residual is not None else None
rms_norm_dynamic_per_token_quant(
ops_out, x, weight, ops_scales, EPS, scale_ub, ops_residual
)
torch.testing.assert_close(ref_scales, ops_scales)
# allow 1 ULP difference
assert (
ref_out.view(torch.uint8).to(torch.int16)
- ops_out.view(torch.uint8).to(torch.int16)
).abs().max() <= 1
if add_residual:
torch.testing.assert_close(ref_residual, ops_residual)
class TestRmsNormDynamicPerTokenQuantIntegration:
def test_kernel_registration_integration(self):
from vllm.kernels.helion.register import get_registered_kernels
registered_kernels = get_registered_kernels()
assert "rms_norm_dynamic_per_token_quant" in registered_kernels
kernel_wrapper = registered_kernels["rms_norm_dynamic_per_token_quant"]
assert kernel_wrapper.op_name == "rms_norm_dynamic_per_token_quant"
assert kernel_wrapper._config_picker is not None
assert kernel_wrapper._mutates_args == ["result", "scale", "residual"]
@@ -1,300 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the rms_norm_per_block_quant helion kernel.
Run `pytest tests/kernels/helion/test_rms_norm_per_block_quant.py`.
"""
import itertools
from typing import Any
import pytest
import torch
from torch._subclasses.fake_tensor import FakeTensorMode
from tests.kernels.helion.utils import skip_if_platform_unsupported
from tests.kernels.quant_utils import FP8_DTYPE
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.ops.rms_norm_per_block_quant import (
_pick_cache,
baseline,
pick_config,
rms_norm_per_block_quant,
)
from vllm.utils.import_utils import has_helion
from vllm.utils.torch_utils import set_random_seed
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
def _generate_fake_input(
num_tokens: int, hidden_size: int, group_size: int
) -> tuple[Any, ...]:
with FakeTensorMode():
input = torch.randn(
(num_tokens, hidden_size), device="cuda", dtype=torch.bfloat16
)
result = torch.empty(input.shape, device=input.device, dtype=FP8_DTYPE)
scale = torch.empty(
(num_tokens, hidden_size // group_size),
device=input.device,
dtype=torch.float32,
)
scale_ub = torch.mean(input).to(scale.dtype)
residual = torch.randn_like(input)
weight = torch.normal(
mean=1.0,
std=1.0,
size=(hidden_size,),
dtype=input.dtype,
device=input.device,
)
epsilon = 1e-6
args = (
result,
input,
weight,
scale,
epsilon,
scale_ub,
residual,
group_size,
False,
)
return args
@pytest.fixture(autouse=True)
def reset_config_manager_singleton():
ConfigManager.reset_instance()
ConfigManager()
yield
ConfigManager.reset_instance()
class TestRmsNormPerBlockQuantConfigPicker:
def setup_method(self):
_pick_cache.clear()
def test_config_picker_exact_match(self):
config_keys = [
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
]
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"hidden_size": 4096, "group_size": 128, "num_tokens": 16}
)
def test_config_picker_closest_match(self):
config_keys = [
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 32}),
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 16}),
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 32}),
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 32}),
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 32}),
]
args = _generate_fake_input(20, 3000, 70)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"hidden_size": 2048, "group_size": 64, "num_tokens": 32}
)
def test_config_picker_no_configs(self):
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
assert selected_key is None
def test_config_picker_fallback_to_largest(self):
config_keys = [
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 32}),
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 16}),
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 32}),
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 32}),
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 32}),
]
args = _generate_fake_input(64, 8192, 256)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"hidden_size": 4096, "group_size": 128, "num_tokens": 32}
)
def test_b200_configs_disable_reduction_warp_specialization(self):
config_set = ConfigManager.get_instance().load_config_set(
"rms_norm_per_block_quant"
)
configs = config_set.to_dict()["nvidia_b200"].values()
# TODO: Remove once the Triton pin includes
# https://github.com/triton-lang/triton/pull/9716. Tracked by
# https://github.com/triton-lang/triton/issues/10901.
assert all(
config["range_warp_specializes"][1] is not True for config in configs
)
DTYPES = [torch.bfloat16, torch.float]
QUANT_DTYPES = [torch.int8, FP8_DTYPE]
VEC_HIDDEN_SIZES = [64, 1024]
# Avoid combinatorial explosion with full Cartesian product
NUM_TOKENS_HIDDEN_SIZES = [
*[(1, i) for i in [64, 128, 1024, 5120]],
*[(2048, i) for i in [64, 1024]],
*[(4096, i) for i in [64]],
]
ADD_RESIDUAL = [False, True]
SCALE_UBS = [True, False]
GROUP_SIZES = [64, 128]
TMA_ALIGNMENTS = [0, 4]
SEEDS = [0]
EPS = 1e-6
class TestRmsNormPerBlockQuantCorrectness:
@pytest.mark.parametrize("num_tokens, hidden_size", NUM_TOKENS_HIDDEN_SIZES)
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
@pytest.mark.parametrize("has_scale_ub", SCALE_UBS)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("quant_dtype", QUANT_DTYPES)
@pytest.mark.parametrize("is_scale_transposed", [False, True])
@pytest.mark.parametrize(
"group_size, tma_alignment",
[*itertools.product(GROUP_SIZES, TMA_ALIGNMENTS)],
)
@pytest.mark.parametrize("seed", SEEDS)
def test_rms_norm_per_block_quant(
self,
num_tokens: int,
hidden_size: int,
add_residual: bool,
has_scale_ub: bool,
dtype: torch.dtype,
quant_dtype: torch.dtype,
is_scale_transposed: bool,
group_size: int,
tma_alignment: int,
seed: int,
) -> None:
skip_if_platform_unsupported("rms_norm_per_block_quant")
set_random_seed(seed)
if hidden_size % group_size != 0:
# skip
return
if tma_alignment != 0 and hidden_size // group_size % tma_alignment == 0:
# Skip tests where TMA alignment doesn't create extra padding to save time
return
if has_scale_ub and quant_dtype != FP8_DTYPE:
# skip
return
scale = 1 / (hidden_size)
input = torch.randn(num_tokens, hidden_size, dtype=dtype, device="cuda") * scale
weight = torch.normal(
mean=1.0, std=1.0, size=(hidden_size,), dtype=dtype, device=input.device
)
residual = torch.randn_like(input) * scale if add_residual else None
scale_ub = (
torch.mean(input).to(dtype=torch.float32, device="cuda")
if has_scale_ub
else None
)
groups_per_row = hidden_size // group_size
ref_residual = residual.clone() if residual is not None else None
ops_residual = residual.clone() if residual is not None else None
ref_out = torch.empty(input.shape, device=input.device, dtype=quant_dtype)
ops_out = ref_out.clone()
if is_scale_transposed:
if tma_alignment == 0:
ref_scales = torch.empty(
(groups_per_row, num_tokens),
device=input.device,
dtype=torch.float32,
).transpose(0, 1)
else:
tma_aligned_m = (
(num_tokens + tma_alignment - 1) // tma_alignment * tma_alignment
)
shape = (num_tokens, groups_per_row)
stride = (1, tma_aligned_m)
ref_scales = torch.empty_strided(
shape, stride, device=input.device, dtype=torch.float32
)
else:
ref_scales = torch.empty(
(num_tokens, groups_per_row), device=input.device, dtype=torch.float32
)
ops_scales = ref_scales.clone()
baseline(
ref_out,
input,
weight,
ref_scales,
EPS,
scale_ub,
ref_residual,
group_size,
is_scale_transposed,
)
ref_scales = ref_scales.contiguous()
rms_norm_per_block_quant(
ops_out,
input,
weight,
ops_scales,
EPS,
scale_ub,
ops_residual,
group_size,
is_scale_transposed,
)
ops_scales = ops_scales.contiguous()
torch.testing.assert_close(ref_scales, ops_scales)
# allow 1 ULP difference
assert (
ref_out.view(torch.uint8).to(torch.int16)
- ops_out.view(torch.uint8).to(torch.int16)
).abs().max() <= 1
if add_residual:
torch.testing.assert_close(ref_residual, ops_residual)
class TestRmsNormPerBlockQuantIntegration:
def test_kernel_registration_integration(self):
from vllm.kernels.helion.register import get_registered_kernels
registered_kernels = get_registered_kernels()
assert "rms_norm_per_block_quant" in registered_kernels
kernel_wrapper = registered_kernels["rms_norm_per_block_quant"]
assert kernel_wrapper.op_name == "rms_norm_per_block_quant"
assert kernel_wrapper._config_picker is not None
assert kernel_wrapper._mutates_args == ["result", "scale", "residual"]
@@ -1,213 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the silu_and_mul_per_block_quant helion kernel
Run `pytest tests/kernels/helion/test_silu_and_mul_per_block_quant.py`.
"""
from typing import Any
import pytest
import torch
from torch._subclasses.fake_tensor import FakeTensorMode
from tests.kernels.helion.utils import skip_if_platform_unsupported
from tests.kernels.quant_utils import FP8_DTYPE
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.ops.silu_and_mul_per_block_quant import (
_pick_cache,
baseline,
pick_config,
silu_and_mul_per_block_quant,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_helion
from vllm.utils.torch_utils import set_random_seed
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
def _generate_fake_input(
num_tokens: int, intermediate_size: int, group_size: int
) -> tuple[Any, ...]:
with FakeTensorMode():
in_dtype: torch.dtype = torch.bfloat16
out_dtype: torch.dtype = current_platform.fp8_dtype()
scale_dtype: torch.dtype = torch.float32
input = torch.randn(
num_tokens, 2 * intermediate_size, device="cuda", dtype=in_dtype
)
result = torch.empty(
num_tokens, intermediate_size, device=input.device, dtype=out_dtype
)
scale = torch.empty(
(num_tokens, intermediate_size // group_size),
device=input.device,
dtype=scale_dtype,
)
scale_ub = torch.mean(input).to(scale_dtype)
args = (
result,
input,
scale,
group_size,
scale_ub,
False,
)
return args
class TestSiluAndMulPerBlockQuantConfigPicker:
def setup_method(self):
_pick_cache.clear()
def test_config_picker_exact_match(self):
config_keys = [
CaseKey({"intermediate_size": 2048, "group_size": 64, "num_tokens": 16}),
CaseKey({"intermediate_size": 4096, "group_size": 128, "num_tokens": 16}),
]
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"intermediate_size": 4096, "group_size": 128, "num_tokens": 16}
)
def test_config_picker_closest_match(self):
config_keys = [
CaseKey({"intermediate_size": 2048, "group_size": 64, "num_tokens": 16}),
CaseKey({"intermediate_size": 2048, "group_size": 64, "num_tokens": 32}),
CaseKey({"intermediate_size": 2048, "group_size": 128, "num_tokens": 16}),
CaseKey({"intermediate_size": 2048, "group_size": 128, "num_tokens": 32}),
CaseKey({"intermediate_size": 4096, "group_size": 64, "num_tokens": 16}),
CaseKey({"intermediate_size": 4096, "group_size": 64, "num_tokens": 32}),
CaseKey({"intermediate_size": 4096, "group_size": 128, "num_tokens": 16}),
CaseKey({"intermediate_size": 4096, "group_size": 128, "num_tokens": 32}),
]
args = _generate_fake_input(20, 3000, 70)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"intermediate_size": 2048, "group_size": 64, "num_tokens": 32}
)
def test_config_picker_no_configs(self):
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
assert selected_key is None
def test_config_picker_fallback_to_largest(self):
config_keys = [
CaseKey({"intermediate_size": 2048, "group_size": 64, "num_tokens": 16}),
CaseKey({"intermediate_size": 2048, "group_size": 64, "num_tokens": 32}),
CaseKey({"intermediate_size": 2048, "group_size": 128, "num_tokens": 16}),
CaseKey({"intermediate_size": 2048, "group_size": 128, "num_tokens": 32}),
CaseKey({"intermediate_size": 4096, "group_size": 64, "num_tokens": 16}),
CaseKey({"intermediate_size": 4096, "group_size": 64, "num_tokens": 32}),
CaseKey({"intermediate_size": 4096, "group_size": 128, "num_tokens": 16}),
CaseKey({"intermediate_size": 4096, "group_size": 128, "num_tokens": 32}),
]
args = _generate_fake_input(64, 8192, 256)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"intermediate_size": 4096, "group_size": 128, "num_tokens": 32}
)
@pytest.fixture(autouse=True)
def reset_config_manager_singleton():
ConfigManager.reset_instance()
ConfigManager()
yield
ConfigManager.reset_instance()
class TestSiluAndMulPerBlockQuantCorrectness:
@pytest.mark.parametrize("num_tokens", [1, 7, 4096])
@pytest.mark.parametrize("hidden_size", [1024, 2048, 5120])
@pytest.mark.parametrize("group_size", [64, 128])
@pytest.mark.parametrize("is_scale_transposed", [False, True])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("quant_dtype", [current_platform.fp8_dtype(), torch.int8])
@pytest.mark.parametrize("has_scale_ub", [True, False])
@pytest.mark.parametrize("seed", [0])
def test_silu_and_mul_per_block_quant(
self,
num_tokens: int,
hidden_size: int,
group_size: int,
is_scale_transposed: bool,
dtype: torch.dtype,
quant_dtype: torch.dtype,
has_scale_ub: bool,
seed: int,
) -> None:
skip_if_platform_unsupported("silu_and_mul_per_block_quant")
set_random_seed(seed)
if hidden_size % group_size != 0:
return
if has_scale_ub and quant_dtype != FP8_DTYPE:
# skip
return
scale = 1 / hidden_size
x = torch.randn(num_tokens, 2 * hidden_size, dtype=dtype, device="cuda") * scale
if has_scale_ub:
act = torch.nn.functional.silu(x[:, :hidden_size]) * x[:, hidden_size:]
act_abs = act.abs().float()
scale_ub = 0.5 * (act_abs.mean() + act_abs.amax())
else:
scale_ub = None
ref_out = torch.empty(num_tokens, hidden_size, device="cuda", dtype=quant_dtype)
if is_scale_transposed:
ref_scales = torch.empty(
(hidden_size // group_size, x.shape[0]),
device="cuda",
dtype=torch.float32,
).t()
else:
ref_scales = torch.empty(
(x.shape[0], hidden_size // group_size),
device="cuda",
dtype=torch.float32,
)
ops_out = ref_out.clone()
ops_scales = ref_scales.clone()
baseline(ref_out, x, ref_scales, group_size, scale_ub, is_scale_transposed)
silu_and_mul_per_block_quant(
ops_out, x, ops_scales, group_size, scale_ub, is_scale_transposed
)
torch.testing.assert_close(ref_scales, ops_scales)
# allow 1 ULP difference
assert (
ref_out.view(torch.uint8).to(torch.int16)
- ops_out.view(torch.uint8).to(torch.int16)
).abs().max() <= 1
class TestSiluAndMulPerBlockQuantIntegration:
def test_kernel_registration_integration(self):
from vllm.kernels.helion.register import get_registered_kernels
registered_kernels = get_registered_kernels()
assert "silu_and_mul_per_block_quant" in registered_kernels
kernel_wrapper = registered_kernels["silu_and_mul_per_block_quant"]
assert kernel_wrapper.op_name == "silu_and_mul_per_block_quant"
assert kernel_wrapper._config_picker is not None
assert kernel_wrapper._mutates_args == ["out", "scales"]
-365
View File
@@ -1,365 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.utils.import_utils import has_helion
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.ops.silu_mul_fp8 import (
_pick_cache,
pick_silu_mul_fp8_config,
silu_mul_fp8,
silu_mul_fp8_baseline,
)
def skip_if_platform_unsupported():
try:
from vllm.kernels.helion.utils import get_canonical_gpu_name
if not torch.cuda.is_available():
pytest.skip("CUDA not available")
platform = get_canonical_gpu_name()
try:
config_manager = ConfigManager.get_instance()
except RuntimeError:
config_manager = ConfigManager()
configs = config_manager.get_platform_configs("silu_mul_fp8", platform)
if len(configs) == 0:
pytest.skip("Current GPU platform not supported for silu_mul_fp8 kernel")
except (ImportError, RuntimeError, KeyError):
pytest.skip("Error detecting platform support for silu_mul_fp8 kernel")
@pytest.fixture(autouse=True)
def reset_config_manager_singleton():
ConfigManager.reset_instance()
ConfigManager()
yield
ConfigManager.reset_instance()
class TestSiluMulFp8ConfigPicker:
def setup_method(self):
_pick_cache.clear()
def test_config_picker_exact_match(self):
config_keys = [
CaseKey({"intermediate": 2048, "numtokens": 256}),
CaseKey({"intermediate": 4096, "numtokens": 256}),
]
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
args = (input_tensor, scale)
selected_key = pick_silu_mul_fp8_config(args, config_keys)
assert selected_key == CaseKey({"intermediate": 2048, "numtokens": 256})
def test_config_picker_closest_match(self):
config_keys = [
CaseKey({"intermediate": 2048, "numtokens": 256}),
CaseKey({"intermediate": 4096, "numtokens": 256}),
]
input_tensor = torch.randn(32, 7000, dtype=torch.bfloat16, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
args = (input_tensor, scale)
selected_key = pick_silu_mul_fp8_config(args, config_keys)
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 256})
def test_config_picker_no_configs(self):
config_keys: list[CaseKey] = []
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
args = (input_tensor, scale)
selected_key = pick_silu_mul_fp8_config(args, config_keys)
assert selected_key is None
@pytest.mark.parametrize("intermediate_size", [2048, 4096, 5120])
def test_config_picker_different_sizes(self, intermediate_size):
config_keys = [
CaseKey({"intermediate": 2048, "numtokens": 256}),
CaseKey({"intermediate": 4096, "numtokens": 256}),
CaseKey({"intermediate": 5120, "numtokens": 256}),
]
input_tensor = torch.randn(
32, 2 * intermediate_size, dtype=torch.bfloat16, device="cuda"
)
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
args = (input_tensor, scale)
selected_key = pick_silu_mul_fp8_config(args, config_keys)
assert selected_key == {
"intermediate": intermediate_size,
"numtokens": 256,
}
def test_config_picker_numtokens_ceiling(self):
config_keys = [
CaseKey({"intermediate": 4096, "numtokens": 8}),
CaseKey({"intermediate": 4096, "numtokens": 32}),
CaseKey({"intermediate": 4096, "numtokens": 128}),
CaseKey({"intermediate": 4096, "numtokens": 256}),
]
input_tensor = torch.randn(20, 8192, dtype=torch.bfloat16, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys)
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 32})
def test_config_picker_numtokens_exact(self):
config_keys = [
CaseKey({"intermediate": 4096, "numtokens": 8}),
CaseKey({"intermediate": 4096, "numtokens": 32}),
CaseKey({"intermediate": 4096, "numtokens": 128}),
]
input_tensor = torch.randn(32, 8192, dtype=torch.bfloat16, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys)
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 32})
def test_config_picker_numtokens_fallback_to_largest(self):
config_keys = [
CaseKey({"intermediate": 4096, "numtokens": 8}),
CaseKey({"intermediate": 4096, "numtokens": 32}),
CaseKey({"intermediate": 4096, "numtokens": 128}),
]
input_tensor = torch.randn(512, 8192, dtype=torch.bfloat16, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys)
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 128})
class TestSiluMulFp8Correctness:
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
@pytest.mark.parametrize("intermediate_size", [2048, 3000, 3500, 4096, 5000])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_silu_mul_fp8_correctness(self, batch_size, intermediate_size, dtype):
skip_if_platform_unsupported()
input_size = 2 * intermediate_size
input_tensor = torch.randn(batch_size, input_size, dtype=dtype, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
reference_output = silu_mul_fp8_baseline(input_tensor, scale)
helion_output = silu_mul_fp8(input_tensor, scale)
assert helion_output.shape == reference_output.shape
assert helion_output.dtype == torch.float8_e4m3fn
assert reference_output.dtype == torch.float8_e4m3fn
ref_f32 = reference_output.to(torch.float32)
helion_f32 = helion_output.to(torch.float32)
# FP8 E4M3 has limited precision. Values near quantization boundaries
# can round differently due to intermediate precision differences.
torch.testing.assert_close(
helion_f32,
ref_f32,
atol=0.05,
rtol=0.05,
msg=f"Mismatch at batch={batch_size}, size={intermediate_size}",
)
def test_silu_mul_fp8_shape_inference(self):
skip_if_platform_unsupported()
batch_size, input_size = 32, 8192
intermediate_size = input_size // 2
input_tensor = torch.randn(
batch_size, input_size, dtype=torch.bfloat16, device="cuda"
)
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
output = silu_mul_fp8(input_tensor, scale)
expected_shape = (batch_size, intermediate_size)
assert output.shape == expected_shape
assert output.dtype == torch.float8_e4m3fn
def test_silu_mul_fp8_scale_variations(self):
skip_if_platform_unsupported()
batch_size, input_size = 16, 4096
input_tensor = torch.randn(
batch_size, input_size, dtype=torch.bfloat16, device="cuda"
)
scales = [0.1, 0.5, 1.0, 2.0, 10.0]
for scale_val in scales:
scale = torch.tensor([scale_val], dtype=torch.float32, device="cuda")
reference_output = silu_mul_fp8_baseline(input_tensor, scale)
helion_output = silu_mul_fp8(input_tensor, scale)
ref_f32 = reference_output.to(torch.float32)
helion_f32 = helion_output.to(torch.float32)
torch.testing.assert_close(
helion_f32,
ref_f32,
atol=0.05,
rtol=0.05,
msg=f"Mismatch for scale={scale_val}",
)
@pytest.mark.parametrize(
"shape",
[
(1, 4096),
(16, 4096),
(128, 4096),
(1024, 4096),
(1, 8192),
(16, 8192),
(128, 8192),
],
)
def test_silu_mul_fp8_various_shapes(self, shape):
skip_if_platform_unsupported()
input_tensor = torch.randn(*shape, dtype=torch.bfloat16, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
reference_output = silu_mul_fp8_baseline(input_tensor, scale)
helion_output = silu_mul_fp8(input_tensor, scale)
assert helion_output.shape == reference_output.shape
ref_f32 = reference_output.to(torch.float32)
helion_f32 = helion_output.to(torch.float32)
torch.testing.assert_close(
helion_f32, ref_f32, atol=0.05, rtol=0.05, msg=f"Mismatch for shape={shape}"
)
def silu_mul_fp8_pytorch(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
"""Pure PyTorch reference using F.silu.
This matches vLLM's SiluAndMul.forward_native exactly:
F.silu(x[..., :d]) * x[..., d:]
"""
d = input.shape[-1] // 2
result = F.silu(input[..., :d]) * input[..., d:]
return (result.to(torch.float32) / scale).to(torch.float8_e4m3fn)
class TestSiluMulFp8PytorchReference:
"""Tests comparing Helion kernel against pure PyTorch implementation.
Uses tighter tolerance since both use PyTorch's FP8 conversion
(same rounding mode), unlike the vLLM C++ baseline which uses
NVIDIA's hardware FP8 conversion with different rounding.
"""
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 256])
@pytest.mark.parametrize("intermediate_size", [1024, 2048, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_silu_mul_fp8_vs_pytorch(self, batch_size, intermediate_size, dtype):
skip_if_platform_unsupported()
input_tensor = torch.randn(
batch_size, 2 * intermediate_size, dtype=dtype, device="cuda"
)
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
pytorch_output = silu_mul_fp8_pytorch(input_tensor, scale)
helion_output = silu_mul_fp8(input_tensor, scale)
assert helion_output.shape == pytorch_output.shape
assert helion_output.dtype == torch.float8_e4m3fn
pytorch_f32 = pytorch_output.to(torch.float32)
helion_f32 = helion_output.to(torch.float32)
# Tolerance accounts for FP8 quantization boundary effects
torch.testing.assert_close(
helion_f32,
pytorch_f32,
atol=0.05,
rtol=0.05,
msg=(
f"Mismatch at batch={batch_size}, size={intermediate_size}, "
f"dtype={dtype}"
),
)
@pytest.mark.parametrize(
"shape",
[
(1, 2, 4096), # 3D input
(2, 4, 2048), # 3D input
(1, 1, 1, 8192), # 4D input
],
)
def test_silu_mul_fp8_multidim_vs_pytorch(self, shape):
skip_if_platform_unsupported()
input_tensor = torch.randn(*shape, dtype=torch.bfloat16, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
pytorch_output = silu_mul_fp8_pytorch(input_tensor, scale)
helion_output = silu_mul_fp8(input_tensor, scale)
assert helion_output.shape == pytorch_output.shape
pytorch_f32 = pytorch_output.to(torch.float32)
helion_f32 = helion_output.to(torch.float32)
torch.testing.assert_close(
helion_f32,
pytorch_f32,
atol=0.05,
rtol=0.05,
msg=f"Mismatch for shape={shape}",
)
class TestSiluMulFp8Integration:
def test_kernel_registration_integration(self):
from vllm.kernels.helion.register import get_registered_kernels
registered_kernels = get_registered_kernels()
assert "silu_mul_fp8" in registered_kernels
kernel_wrapper = registered_kernels["silu_mul_fp8"]
assert kernel_wrapper.op_name == "silu_mul_fp8"
assert kernel_wrapper._config_picker is not None
def test_fake_impl_functionality(self):
skip_if_platform_unsupported()
from vllm.kernels.helion.register import get_registered_kernels
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
registered_kernels = get_registered_kernels()
kernel_wrapper = registered_kernels["silu_mul_fp8"]
fake_impl = kernel_wrapper._fake_impl
assert fake_impl is not None
fake_output = fake_impl(input_tensor, scale)
expected_shape = (32, 2048)
assert fake_output.shape == expected_shape
assert fake_output.dtype == torch.float8_e4m3fn
assert fake_output.device == input_tensor.device
-33
View File
@@ -1,33 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for Helion utility functions."""
import pytest
from vllm.kernels.helion.utils import canonicalize_gpu_name
@pytest.mark.parametrize(
"driver_reported_name,expected",
[
("NVIDIA H200", "nvidia_h200"),
("NVIDIA A100-SXM4-80GB", "nvidia_a100"),
("NVIDIA H100 80GB HBM3", "nvidia_h100"),
("NVIDIA H100 PCIe", "nvidia_h100"),
("NVIDIA H100 SXM5", "nvidia_h100"),
("NVIDIA GeForce RTX 4090", "nvidia_geforce_rtx_4090"),
("AMD Instinct MI300X", "amd_instinct_mi300x"),
("AMD Instinct MI250X / MI250", "amd_instinct_mi250x_mi250"),
("Tesla V100-SXM2-32GB", "tesla_v100"),
],
)
def test_canonicalize_gpu_name(driver_reported_name, expected):
"""Test GPU name canonicalization."""
assert canonicalize_gpu_name(driver_reported_name) == expected
@pytest.mark.parametrize("invalid_name", ["", " ", "\t", "\n"])
def test_canonicalize_gpu_name_rejects_empty(invalid_name):
"""Test that empty or whitespace-only names are rejected."""
with pytest.raises(ValueError, match="cannot be empty"):
canonicalize_gpu_name(invalid_name)
-30
View File
@@ -1,30 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helion Kernel test utils."""
import pytest
import torch
from vllm.kernels.helion.config_manager import ConfigManager
def skip_if_platform_unsupported(op_name: str):
try:
from vllm.kernels.helion.utils import get_canonical_gpu_name
if not torch.cuda.is_available():
pytest.skip("CUDA not available")
platform = get_canonical_gpu_name()
try:
config_manager = ConfigManager.get_instance()
except RuntimeError:
config_manager = ConfigManager()
configs = config_manager.get_platform_configs(op_name, platform)
if len(configs) == 0:
pytest.skip(f"Current GPU platform not supported for {op_name} kernel")
except (ImportError, RuntimeError, KeyError):
pytest.skip(f"Error detecting platform support for {op_name} kernel")