[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,75 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import helion
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.register import register_kernel
|
||||
from vllm.kernels.helion.utils import get_canonical_gpu_name
|
||||
|
||||
GPU_PLATFORM = get_canonical_gpu_name()
|
||||
|
||||
DEFAULT_CONFIGS: dict[CaseKey, helion.Config] = {
|
||||
CaseKey.default(): helion.Config(block_sizes=[32]),
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def dummy_kernel_registry(
|
||||
configs: dict[CaseKey, helion.Config] | None = None,
|
||||
):
|
||||
"""Context manager providing a register function with automatic config setup.
|
||||
|
||||
Yields a ``register`` callable with the same signature as
|
||||
``register_kernel``. Before applying the real decorator it writes a
|
||||
config JSON for the kernel name (from ``op_name`` or ``fn.__name__``)
|
||||
into a temporary directory backed by a fresh ``ConfigManager``.
|
||||
"""
|
||||
if configs is None:
|
||||
configs = DEFAULT_CONFIGS
|
||||
|
||||
def _to_config_entries(cfgs: dict) -> list[dict[str, Any]]:
|
||||
pairs: list[dict[str, Any]] = []
|
||||
for k, v in cfgs.items():
|
||||
config_data = v.__dict__["config"]
|
||||
pairs.append({"key": dict(k), "config": config_data})
|
||||
return pairs
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
config_dir = Path(tmpdir)
|
||||
ConfigManager.reset_instance()
|
||||
cm = ConfigManager(base_dir=config_dir)
|
||||
|
||||
with patch(
|
||||
"vllm.kernels.helion.config_manager.ConfigManager",
|
||||
return_value=cm,
|
||||
):
|
||||
|
||||
def register(
|
||||
op_name: str | None = None,
|
||||
**kwargs,
|
||||
) -> Callable:
|
||||
def decorator(fn: Callable) -> Callable:
|
||||
name = op_name or fn.__name__
|
||||
kernel_dir = config_dir / name
|
||||
kernel_dir.mkdir(parents=True, exist_ok=True)
|
||||
(kernel_dir / f"{GPU_PLATFORM}.json").write_text(
|
||||
json.dumps(_to_config_entries(configs))
|
||||
)
|
||||
return register_kernel(op_name, **kwargs)(fn)
|
||||
|
||||
return decorator
|
||||
|
||||
try:
|
||||
yield register
|
||||
finally:
|
||||
ConfigManager.reset_instance()
|
||||
@@ -1,91 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for autotuning Helion kernels, including disabled kernels with no configs."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
pytest.skip(
|
||||
"Helion is not installed. Install with: pip install vllm[helion]",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
import helion
|
||||
import helion.language as hl
|
||||
from helion.autotuner.base_search import BaseSearch
|
||||
|
||||
from tests.kernels.helion.helpers import dummy_kernel_registry
|
||||
from vllm.kernels.helion.register import create_helion_decorated_kernel
|
||||
|
||||
|
||||
def _add_kernel(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
out = torch.empty_like(x)
|
||||
for tile in hl.tile(x.size()):
|
||||
out[tile] = x[tile] + y[tile]
|
||||
return out
|
||||
|
||||
|
||||
class NoCompileSearch(BaseSearch):
|
||||
"""Autotuner that returns the default config without GPU compilation.
|
||||
|
||||
Modeled after helion's test BasicSearch (pytorch/helion#1649).
|
||||
"""
|
||||
|
||||
def autotune(self, *, skip_cache: bool = False):
|
||||
return self.config_spec.default_config()
|
||||
|
||||
|
||||
def _no_compile_autotuner_fn(bound_kernel, args, **kwargs):
|
||||
return NoCompileSearch(bound_kernel, args, **kwargs)
|
||||
|
||||
|
||||
class TestAutotuneDisabledKernel:
|
||||
"""Test autotuning flow on disabled kernels (no platform configs)."""
|
||||
|
||||
def setup_method(self):
|
||||
from vllm.kernels.helion.register import _REGISTERED_KERNELS
|
||||
|
||||
self._saved_registry = dict(_REGISTERED_KERNELS)
|
||||
_REGISTERED_KERNELS.clear()
|
||||
|
||||
def teardown_method(self):
|
||||
from vllm.kernels.helion.register import _REGISTERED_KERNELS
|
||||
|
||||
_REGISTERED_KERNELS.clear()
|
||||
_REGISTERED_KERNELS.update(self._saved_registry)
|
||||
|
||||
def test_autotune_disabled_kernel_produces_valid_config(self):
|
||||
"""Register a kernel with no configs (disabled), run autotune,
|
||||
verify it produces a valid helion.Config."""
|
||||
with dummy_kernel_registry(configs={}) as register:
|
||||
wrapper = register(
|
||||
"autotune_test_kernel",
|
||||
config_picker=lambda args, keys: None,
|
||||
fake_impl=lambda *a, **kw: None,
|
||||
input_generator=lambda: {
|
||||
"small": (
|
||||
torch.randn(4, 4, device="cuda"),
|
||||
torch.randn(4, 4, device="cuda"),
|
||||
),
|
||||
},
|
||||
)(_add_kernel)
|
||||
|
||||
assert wrapper._disabled is True
|
||||
|
||||
inputs = wrapper.get_inputs()
|
||||
assert "small" in inputs
|
||||
|
||||
settings = helion.Settings()
|
||||
settings.autotuner_fn = _no_compile_autotuner_fn
|
||||
wrapper.helion_settings = settings
|
||||
|
||||
config = wrapper.run_autotune(inputs["small"])
|
||||
expected_default = (
|
||||
create_helion_decorated_kernel(_add_kernel, helion_settings=settings)
|
||||
.bind(inputs["small"])
|
||||
.config_spec.default_config()
|
||||
)
|
||||
assert config == expected_default
|
||||
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
pytest.skip(
|
||||
"Helion is not installed. Install with: pip install vllm[helion]",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
|
||||
|
||||
class TestCaseKey:
|
||||
"""Test suite for CaseKey class."""
|
||||
|
||||
def test_construction_with_dict(self):
|
||||
key = CaseKey({"intermediate": 2048, "numtokens": 256})
|
||||
assert key["intermediate"] == 2048
|
||||
assert key["numtokens"] == 256
|
||||
|
||||
def test_empty_construction_raises(self):
|
||||
with pytest.raises(TypeError, match="at least one key-value pair"):
|
||||
CaseKey()
|
||||
with pytest.raises(TypeError, match="at least one key-value pair"):
|
||||
CaseKey({})
|
||||
|
||||
def test_default_construction(self):
|
||||
key = CaseKey.default()
|
||||
assert len(key) == 0
|
||||
assert key.is_default()
|
||||
|
||||
def test_non_default_is_not_default(self):
|
||||
key = CaseKey({"intermediate": 2048})
|
||||
assert not key.is_default()
|
||||
|
||||
def test_hashable_and_equality(self):
|
||||
a = CaseKey({"intermediate": 2048, "numtokens": 256})
|
||||
b = CaseKey({"numtokens": 256, "intermediate": 2048})
|
||||
assert a == b
|
||||
assert hash(a) == hash(b)
|
||||
assert a != CaseKey({"intermediate": 4096})
|
||||
assert CaseKey.default() == CaseKey.default()
|
||||
|
||||
configs = {
|
||||
CaseKey.default(): "default_config",
|
||||
a: "a_config",
|
||||
}
|
||||
assert configs[b] == "a_config"
|
||||
assert configs[CaseKey.default()] == "default_config"
|
||||
|
||||
def test_str_is_sorted_json(self):
|
||||
assert str(CaseKey({"z": 1, "a": 2})) == '{"a":2,"z":1}'
|
||||
assert str(CaseKey.default()) == "{}"
|
||||
|
||||
def test_immutable(self):
|
||||
key = CaseKey({"intermediate": 2048})
|
||||
with pytest.raises(TypeError, match="immutable"):
|
||||
key["intermediate"] = 4096
|
||||
with pytest.raises(TypeError, match="immutable"):
|
||||
del key["intermediate"]
|
||||
with pytest.raises(TypeError, match="immutable"):
|
||||
key.update({"numtokens": 256})
|
||||
with pytest.raises(TypeError, match="immutable"):
|
||||
key.clear()
|
||||
@@ -1,408 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for Helion ConfigManager and ConfigSet.
|
||||
|
||||
Tests the simplified configuration management system for Helion custom kernels.
|
||||
"""
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
# Skip entire module if helion is not available
|
||||
if not has_helion():
|
||||
pytest.skip(
|
||||
"Helion is not installed. Install with: pip install vllm[helion]",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
import helion
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import (
|
||||
ConfigManager,
|
||||
ConfigSet,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_config_manager_singleton():
|
||||
"""Reset ConfigManager singleton before each test."""
|
||||
ConfigManager.reset_instance()
|
||||
yield
|
||||
ConfigManager.reset_instance()
|
||||
|
||||
|
||||
class TestConfigSet:
|
||||
"""Test suite for ConfigSet class."""
|
||||
|
||||
def test_config_set_creation(self):
|
||||
"""Test creating an empty ConfigSet."""
|
||||
config_set = ConfigSet("test_kernel")
|
||||
|
||||
assert config_set.kernel_name == "test_kernel"
|
||||
assert config_set.get_platforms() == []
|
||||
|
||||
def test_config_set_from_dict(self):
|
||||
"""Test creating ConfigSet from dictionary data."""
|
||||
config_data = {
|
||||
"block_sizes": [32, 16],
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"pid_type": "persistent_interleaved",
|
||||
}
|
||||
data = {
|
||||
"h100": [
|
||||
{"key": {"batch": 32, "hidden": 4096}, "config": config_data},
|
||||
]
|
||||
}
|
||||
|
||||
config_set = ConfigSet.from_dict("test_kernel", data)
|
||||
|
||||
assert config_set.kernel_name == "test_kernel"
|
||||
assert config_set.get_platforms() == ["h100"]
|
||||
|
||||
internal_key = CaseKey({"batch": 32, "hidden": 4096})
|
||||
config = config_set.get_config("h100", internal_key)
|
||||
assert isinstance(config, helion.Config)
|
||||
assert config.block_sizes == [32, 16]
|
||||
assert config.num_warps == 4
|
||||
assert config.num_stages == 3
|
||||
assert config.pid_type == "persistent_interleaved"
|
||||
|
||||
def test_config_set_get_config_keyerror(self):
|
||||
"""Test that accessing non-existent configs raises informative KeyErrors."""
|
||||
config_set = ConfigSet("test_kernel")
|
||||
|
||||
with pytest.raises(KeyError, match="platform 'h100' not found"):
|
||||
config_set.get_config("h100", CaseKey({"batch": 32}))
|
||||
|
||||
config_data = {"num_warps": 8, "num_stages": 4}
|
||||
data = {
|
||||
"h100": [
|
||||
{"key": {"batch": 64, "hidden": 2048}, "config": config_data},
|
||||
]
|
||||
}
|
||||
config_set = ConfigSet.from_dict("test_kernel", data)
|
||||
|
||||
nonexistent_key = CaseKey({"batch": 32, "hidden": 4096})
|
||||
with pytest.raises(KeyError, match="config_key .* not found"):
|
||||
config_set.get_config("h100", nonexistent_key)
|
||||
|
||||
def test_config_set_get_platforms(self):
|
||||
"""Test get_platforms method."""
|
||||
# Use realistic config data
|
||||
config1 = {"num_warps": 4, "num_stages": 3}
|
||||
config2 = {"num_warps": 8, "num_stages": 5}
|
||||
|
||||
data = {
|
||||
"h100": [
|
||||
{"key": {"batch": 32, "hidden": 4096}, "config": config1},
|
||||
],
|
||||
"a100": [
|
||||
{"key": {"batch": 16, "hidden": 2048}, "config": config2},
|
||||
],
|
||||
}
|
||||
config_set = ConfigSet.from_dict("test_kernel", data)
|
||||
|
||||
platforms = config_set.get_platforms()
|
||||
assert platforms == ["a100", "h100"] # Should be sorted
|
||||
|
||||
def test_config_set_get_config_keys(self):
|
||||
"""Test get_config_keys method."""
|
||||
config1 = {"num_warps": 4, "num_stages": 3}
|
||||
config2 = {"num_warps": 8, "num_stages": 5}
|
||||
|
||||
data = {
|
||||
"h100": [
|
||||
{"key": {"batch": 32, "hidden": 4096}, "config": config1},
|
||||
{"key": {"batch": 64, "hidden": 2048}, "config": config2},
|
||||
]
|
||||
}
|
||||
config_set = ConfigSet.from_dict("test_kernel", data)
|
||||
|
||||
config_keys = config_set.get_config_keys("h100")
|
||||
expected_keys = sorted(
|
||||
[
|
||||
CaseKey({"batch": 32, "hidden": 4096}),
|
||||
CaseKey({"batch": 64, "hidden": 2048}),
|
||||
],
|
||||
key=lambda k: str(k) if k is not None else "",
|
||||
)
|
||||
assert config_keys == expected_keys
|
||||
|
||||
assert config_set.get_config_keys("v100") == []
|
||||
|
||||
def test_config_set_to_dict(self):
|
||||
"""Test converting ConfigSet to dictionary."""
|
||||
original_config = {
|
||||
"block_sizes": [64, 32],
|
||||
"num_warps": 16,
|
||||
"num_stages": 4,
|
||||
"pid_type": "persistent_blocked",
|
||||
}
|
||||
original_data = {
|
||||
"h100": [
|
||||
{"key": {"batch": 32, "hidden": 4096}, "config": original_config},
|
||||
]
|
||||
}
|
||||
|
||||
config_set = ConfigSet.from_dict("test_kernel", original_data)
|
||||
result_data = config_set.to_dict()
|
||||
|
||||
internal_key = CaseKey({"batch": 32, "hidden": 4096})
|
||||
assert internal_key in result_data["h100"]
|
||||
assert result_data["h100"][internal_key] == original_config
|
||||
|
||||
|
||||
class TestConfigManager:
|
||||
"""Test suite for ConfigManager class."""
|
||||
|
||||
def test_config_manager_creation_default_base_dir(self):
|
||||
"""Test creating ConfigManager with default base directory."""
|
||||
manager = ConfigManager()
|
||||
assert manager._base_dir.name == "configs"
|
||||
|
||||
def test_config_manager_creation_custom_base_dir(self):
|
||||
"""Test creating ConfigManager with custom base directory."""
|
||||
custom_dir = "/tmp/custom_configs"
|
||||
manager = ConfigManager(base_dir=custom_dir)
|
||||
|
||||
# Paths are resolved, so compare with resolved path
|
||||
assert manager._base_dir == Path(custom_dir).resolve()
|
||||
|
||||
def test_get_config_file_path(self):
|
||||
"""Test getting config file path for a kernel."""
|
||||
manager = ConfigManager(base_dir="/tmp")
|
||||
|
||||
dir_path = manager.get_config_file_path("silu_mul_fp8")
|
||||
assert dir_path == Path("/tmp/silu_mul_fp8")
|
||||
|
||||
file_path = manager.get_config_file_path("silu_mul_fp8", "nvidia_h100")
|
||||
assert file_path == Path("/tmp/silu_mul_fp8/nvidia_h100.json")
|
||||
|
||||
def test_ensure_base_dir_exists(self):
|
||||
"""Test ensuring base directory exists."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
base_dir = Path(temp_dir) / "non_existent" / "configs"
|
||||
manager = ConfigManager(base_dir=base_dir)
|
||||
assert not base_dir.exists()
|
||||
|
||||
returned_path = manager.ensure_base_dir_exists()
|
||||
|
||||
assert base_dir.exists()
|
||||
assert base_dir.is_dir()
|
||||
assert returned_path == base_dir
|
||||
|
||||
def test_load_config_set_file_not_exists(self):
|
||||
"""Test loading config set when file doesn't exist."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
manager = ConfigManager(base_dir=temp_dir)
|
||||
config_set = manager.load_config_set("non_existent_kernel")
|
||||
|
||||
assert isinstance(config_set, ConfigSet)
|
||||
assert config_set.kernel_name == "non_existent_kernel"
|
||||
assert config_set.get_platforms() == []
|
||||
|
||||
def test_load_config_set_valid_file(self):
|
||||
"""Test loading config set from per-platform files."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
kernel_config = {
|
||||
"block_sizes": [128, 64],
|
||||
"num_warps": 8,
|
||||
"num_stages": 6,
|
||||
"pid_type": "persistent_interleaved",
|
||||
}
|
||||
kernel_dir = Path(temp_dir) / "test_kernel"
|
||||
kernel_dir.mkdir()
|
||||
platform_file = kernel_dir / "h100.json"
|
||||
with open(platform_file, "w") as f:
|
||||
json.dump(
|
||||
[{"key": {"batch": 32, "hidden": 4096}, "config": kernel_config}],
|
||||
f,
|
||||
)
|
||||
|
||||
manager = ConfigManager(base_dir=temp_dir)
|
||||
config_set = manager.load_config_set("test_kernel")
|
||||
|
||||
assert isinstance(config_set, ConfigSet)
|
||||
assert config_set.kernel_name == "test_kernel"
|
||||
assert config_set.get_platforms() == ["h100"]
|
||||
|
||||
internal_key = CaseKey({"batch": 32, "hidden": 4096})
|
||||
config = config_set.get_config("h100", internal_key)
|
||||
assert isinstance(config, helion.Config)
|
||||
assert config.block_sizes == [128, 64]
|
||||
assert config.num_warps == 8
|
||||
|
||||
def test_load_config_set_invalid_json(self):
|
||||
"""Test loading config set from file with invalid JSON."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
kernel_dir = Path(temp_dir) / "test_kernel"
|
||||
kernel_dir.mkdir()
|
||||
config_file = kernel_dir / "h100.json"
|
||||
with open(config_file, "w") as f:
|
||||
f.write("invalid json content {")
|
||||
|
||||
manager = ConfigManager(base_dir=temp_dir)
|
||||
config_set = manager.load_config_set("test_kernel")
|
||||
|
||||
assert isinstance(config_set, ConfigSet)
|
||||
assert config_set.kernel_name == "test_kernel"
|
||||
assert config_set.get_platforms() == []
|
||||
|
||||
def test_save_config_set(self):
|
||||
"""Test saving ConfigSet to per-platform files."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
kernel_config = {
|
||||
"block_sizes": [256, 128],
|
||||
"num_warps": 16,
|
||||
"num_stages": 8,
|
||||
"pid_type": "persistent_blocked",
|
||||
}
|
||||
data = {
|
||||
"h100": [
|
||||
{"key": {"batch": 32, "hidden": 4096}, "config": kernel_config},
|
||||
]
|
||||
}
|
||||
config_set = ConfigSet.from_dict("test_kernel", data)
|
||||
|
||||
manager = ConfigManager(base_dir=temp_dir)
|
||||
saved_path = manager.save_config_set(config_set)
|
||||
|
||||
expected_dir = Path(temp_dir) / "test_kernel"
|
||||
assert saved_path == expected_dir
|
||||
assert saved_path.is_dir()
|
||||
|
||||
platform_file = expected_dir / "h100.json"
|
||||
assert platform_file.exists()
|
||||
with open(platform_file) as f:
|
||||
loaded_data = json.load(f)
|
||||
assert isinstance(loaded_data, list)
|
||||
assert len(loaded_data) == 1
|
||||
entry = loaded_data[0]
|
||||
assert entry["key"] == {"batch": 32, "hidden": 4096}
|
||||
assert entry["config"] == kernel_config
|
||||
|
||||
def test_save_config_set_creates_directory(self):
|
||||
"""Test that save_config_set creates parent directories if needed."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
nested_dir = Path(temp_dir) / "nested" / "configs"
|
||||
data = {
|
||||
"h100": [
|
||||
{"key": {}, "config": {"num_warps": 4}},
|
||||
]
|
||||
}
|
||||
config_set = ConfigSet.from_dict("test_kernel", data)
|
||||
|
||||
manager = ConfigManager(base_dir=nested_dir)
|
||||
saved_path = manager.save_config_set(config_set)
|
||||
|
||||
assert nested_dir.exists()
|
||||
assert nested_dir.is_dir()
|
||||
assert saved_path.is_dir()
|
||||
assert (saved_path / "h100.json").exists()
|
||||
|
||||
def test_get_platform_configs(self):
|
||||
"""Test getting all configs for a specific platform."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
config_1 = {"num_warps": 4, "num_stages": 3, "block_sizes": [64, 32]}
|
||||
config_2 = {"num_warps": 8, "num_stages": 5, "block_sizes": [128, 64]}
|
||||
default_config = {
|
||||
"num_warps": 16,
|
||||
"num_stages": 7,
|
||||
"block_sizes": [256, 128],
|
||||
}
|
||||
config_3 = {"num_warps": 2, "num_stages": 2, "block_sizes": [32, 16]}
|
||||
|
||||
kernel_dir = Path(temp_dir) / "test_kernel"
|
||||
kernel_dir.mkdir()
|
||||
with open(kernel_dir / "h100.json", "w") as f:
|
||||
json.dump(
|
||||
[
|
||||
{"key": {"batch": 32, "hidden": 4096}, "config": config_1},
|
||||
{"key": {"batch": 64, "hidden": 2048}, "config": config_2},
|
||||
{"key": {}, "config": default_config},
|
||||
],
|
||||
f,
|
||||
)
|
||||
with open(kernel_dir / "a100.json", "w") as f:
|
||||
json.dump(
|
||||
[{"key": {"batch": 16, "hidden": 1024}, "config": config_3}],
|
||||
f,
|
||||
)
|
||||
|
||||
manager = ConfigManager(base_dir=temp_dir)
|
||||
|
||||
key_b32_h4096 = CaseKey({"batch": 32, "hidden": 4096})
|
||||
key_b64_h2048 = CaseKey({"batch": 64, "hidden": 2048})
|
||||
key_b16_h1024 = CaseKey({"batch": 16, "hidden": 1024})
|
||||
|
||||
h100_configs = manager.get_platform_configs("test_kernel", "h100")
|
||||
assert len(h100_configs) == 3
|
||||
assert key_b32_h4096 in h100_configs
|
||||
assert key_b64_h2048 in h100_configs
|
||||
assert CaseKey.default() in h100_configs
|
||||
for config in h100_configs.values():
|
||||
assert isinstance(config, helion.Config)
|
||||
|
||||
assert h100_configs[key_b32_h4096].num_warps == 4
|
||||
assert h100_configs[CaseKey.default()].num_stages == 7
|
||||
|
||||
a100_configs = manager.get_platform_configs("test_kernel", "a100")
|
||||
assert len(a100_configs) == 1
|
||||
assert key_b16_h1024 in a100_configs
|
||||
assert isinstance(a100_configs[key_b16_h1024], helion.Config)
|
||||
assert a100_configs[key_b16_h1024].num_warps == 2
|
||||
|
||||
nonexistent_configs = manager.get_platform_configs("test_kernel", "v100")
|
||||
assert len(nonexistent_configs) == 0
|
||||
|
||||
def test_singleton_returns_same_instance(self):
|
||||
"""Test that ConfigManager returns the same instance on repeated calls."""
|
||||
manager1 = ConfigManager(base_dir="/tmp/test_singleton")
|
||||
manager2 = ConfigManager(base_dir="/tmp/test_singleton")
|
||||
|
||||
assert manager1 is manager2
|
||||
|
||||
def test_singleton_with_default_base_dir(self):
|
||||
"""Test singleton behavior with default base directory."""
|
||||
manager1 = ConfigManager()
|
||||
manager2 = ConfigManager()
|
||||
|
||||
assert manager1 is manager2
|
||||
assert manager1._base_dir == manager2._base_dir
|
||||
|
||||
def test_singleton_error_on_different_base_dir(self):
|
||||
"""Test that ConfigManager raises error when created with different base_dir."""
|
||||
ConfigManager(base_dir="/tmp/first_dir")
|
||||
|
||||
with pytest.raises(ValueError, match="singleton already exists"):
|
||||
ConfigManager(base_dir="/tmp/different_dir")
|
||||
|
||||
def test_reset_instance_allows_new_base_dir(self):
|
||||
"""Test that reset_instance allows creating with a new base_dir."""
|
||||
manager1 = ConfigManager(base_dir="/tmp/first_dir")
|
||||
assert manager1._base_dir == Path("/tmp/first_dir").resolve()
|
||||
|
||||
ConfigManager.reset_instance()
|
||||
|
||||
manager2 = ConfigManager(base_dir="/tmp/second_dir")
|
||||
assert manager2._base_dir == Path("/tmp/second_dir").resolve()
|
||||
assert manager1 is not manager2
|
||||
|
||||
def test_get_instance_returns_existing(self):
|
||||
"""Test that get_instance returns the existing singleton."""
|
||||
manager1 = ConfigManager(base_dir="/tmp/test_get_instance")
|
||||
manager2 = ConfigManager.get_instance()
|
||||
|
||||
assert manager1 is manager2
|
||||
|
||||
def test_get_instance_raises_if_not_initialized(self):
|
||||
"""Test that get_instance raises RuntimeError if no instance exists."""
|
||||
with pytest.raises(RuntimeError, match="has not been created"):
|
||||
ConfigManager.get_instance()
|
||||
@@ -1,154 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the dynamic_per_token_scaled_fp8_quant helion kernel.
|
||||
|
||||
Run `pytest tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py`.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
from tests.kernels.helion.utils import skip_if_platform_unsupported
|
||||
from tests.kernels.quant_utils import FP8_DTYPE
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.ops.dynamic_per_token_scaled_fp8_quant import (
|
||||
_pick_cache,
|
||||
baseline,
|
||||
dynamic_per_token_scaled_fp8_quant,
|
||||
pick_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_helion
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not has_helion():
|
||||
pytest.skip(
|
||||
"Helion is not installed. Install with: pip install vllm[helion]",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def _generate_fake_input(num_tokens: int, hidden_size: int) -> tuple[Any, ...]:
|
||||
with FakeTensorMode():
|
||||
input = torch.randn(
|
||||
num_tokens, hidden_size, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
result = torch.empty(
|
||||
input.shape, device=input.device, dtype=current_platform.fp8_dtype()
|
||||
)
|
||||
scale = torch.empty((num_tokens, 1), device=input.device, dtype=torch.float32)
|
||||
scale_ub = torch.mean(input).to(torch.float32)
|
||||
args = (result, input, scale, scale_ub)
|
||||
return args
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_config_manager_singleton():
|
||||
ConfigManager.reset_instance()
|
||||
ConfigManager()
|
||||
yield
|
||||
ConfigManager.reset_instance()
|
||||
|
||||
|
||||
class TestDynamicPerTokenScaledFp8QuantConfigPicker:
|
||||
def setup_method(self):
|
||||
_pick_cache.clear()
|
||||
|
||||
def test_config_picker_exact_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(16, 4096)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey({"hidden_size": 4096, "num_tokens": 16})
|
||||
|
||||
def test_config_picker_closest_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 2048, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "num_tokens": 32}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(20, 3000)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key is None
|
||||
|
||||
def test_config_picker_fallback_to_largest(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(32, 8192)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey({"hidden_size": 4096, "num_tokens": 16})
|
||||
|
||||
|
||||
class TestDynamicPerTokenScaledFp8QuantCorrectness:
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 4096])
|
||||
@pytest.mark.parametrize("hidden_size", [17, 1024, 1025, 1026, 5137, 8193])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float])
|
||||
@pytest.mark.parametrize("has_scale_ub", [True, False])
|
||||
@pytest.mark.parametrize("seed", [0])
|
||||
def test_dynamic_per_token_fp8_quant(
|
||||
self,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
has_scale_ub: bool,
|
||||
seed: int,
|
||||
) -> None:
|
||||
skip_if_platform_unsupported("dynamic_per_token_scaled_fp8_quant")
|
||||
set_random_seed(seed)
|
||||
|
||||
x = (
|
||||
torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") + 1e-6
|
||||
) # avoid nans
|
||||
|
||||
scale_ub = (
|
||||
torch.mean(x).to(dtype=torch.float32, device="cuda")
|
||||
if has_scale_ub
|
||||
else None
|
||||
)
|
||||
|
||||
ref_out = torch.empty(x.shape, device="cuda", dtype=FP8_DTYPE)
|
||||
ref_scales = torch.empty((x.shape[0], 1), device="cuda", dtype=torch.float32)
|
||||
baseline(ref_out, x, ref_scales, scale_ub)
|
||||
|
||||
ops_out = torch.empty(x.shape, device="cuda", dtype=FP8_DTYPE)
|
||||
ops_scales = torch.empty((x.shape[0], 1), device="cuda", dtype=torch.float32)
|
||||
dynamic_per_token_scaled_fp8_quant(ops_out, x, ops_scales, scale_ub)
|
||||
|
||||
torch.testing.assert_close(ref_scales, ops_scales)
|
||||
# allow 1 ULP difference
|
||||
assert (
|
||||
ref_out.view(torch.uint8).to(torch.int16)
|
||||
- ops_out.view(torch.uint8).to(torch.int16)
|
||||
).abs().max() <= 1
|
||||
|
||||
|
||||
class TestDynamicPerTokenScaledFp8QuantIntegration:
|
||||
def test_kernel_registration_integration(self):
|
||||
from vllm.kernels.helion.register import get_registered_kernels
|
||||
|
||||
registered_kernels = get_registered_kernels()
|
||||
assert "dynamic_per_token_scaled_fp8_quant" in registered_kernels
|
||||
|
||||
kernel_wrapper = registered_kernels["dynamic_per_token_scaled_fp8_quant"]
|
||||
assert kernel_wrapper.op_name == "dynamic_per_token_scaled_fp8_quant"
|
||||
assert kernel_wrapper._config_picker is not None
|
||||
assert kernel_wrapper._mutates_args == ["result", "scale"]
|
||||
@@ -1,206 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test make_fx tracing and inductor pattern matching with HelionKernelWrapper."""
|
||||
|
||||
import contextlib
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
pytest.skip(
|
||||
"Helion is not installed. Install with: pip install vllm[helion]",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
import helion
|
||||
import helion.language as hl
|
||||
from helion._compat import requires_torch_version
|
||||
|
||||
if not requires_torch_version("2.11"):
|
||||
pytest.skip(
|
||||
"HigherOrderOp requires PyTorch >= 2.11",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from helion._compiler._dynamo.higher_order_ops import (
|
||||
helion_kernel_side_table,
|
||||
helion_kernel_wrapper_mutation,
|
||||
)
|
||||
from torch._inductor.pattern_matcher import (
|
||||
PatternMatcherPass,
|
||||
fwd_only,
|
||||
register_replacement,
|
||||
select_decomp_table,
|
||||
)
|
||||
from torch.fx.experimental.proxy_tensor import make_fx
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.register import HelionKernelWrapper
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _helion_mock_context():
|
||||
configs = {
|
||||
CaseKey.default(): helion.Config(block_sizes=[64], num_warps=2, num_stages=2),
|
||||
}
|
||||
mock_config_manager = Mock(spec=ConfigManager)
|
||||
mock_config_manager.get_platform_configs = Mock(return_value=configs)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.kernels.helion.config_manager.ConfigManager",
|
||||
return_value=mock_config_manager,
|
||||
),
|
||||
patch(
|
||||
"vllm.kernels.helion.utils.get_canonical_gpu_name",
|
||||
return_value="nvidia_h200",
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
class TestMakeFxHop:
|
||||
def setup_method(self):
|
||||
helion_kernel_side_table.reset_table()
|
||||
|
||||
@pytest.mark.skip(reason="SymInt proxy tracking issue with PyTorch 2.11+")
|
||||
def test_make_fx_symbolic(self):
|
||||
def raw_add_scale(
|
||||
x: torch.Tensor, y: torch.Tensor, scale: float
|
||||
) -> tuple[torch.Tensor, int, torch.Tensor]:
|
||||
out_x = torch.empty_like(x)
|
||||
out_y = torch.empty_like(x)
|
||||
for tile in hl.tile(x.size()):
|
||||
out_x[tile] = x[tile] + y[tile] * scale
|
||||
out_y[tile] = out_x[tile] * 2.0
|
||||
return out_x, 42, out_y
|
||||
|
||||
input_x = torch.randn(7, 13)
|
||||
input_y = torch.randn(7, 13)
|
||||
scale = 0.5
|
||||
|
||||
with _helion_mock_context():
|
||||
wrapper = HelionKernelWrapper(
|
||||
raw_kernel_func=raw_add_scale,
|
||||
op_name="test_make_fx",
|
||||
fake_impl=lambda *a, **kw: None,
|
||||
config_picker=lambda args, keys: CaseKey.default(),
|
||||
)
|
||||
|
||||
def fn(x, y):
|
||||
return wrapper(x, y, scale)
|
||||
|
||||
gm = make_fx(fn, tracing_mode="symbolic")(input_x, input_y)
|
||||
|
||||
hop_nodes = [
|
||||
n
|
||||
for n in gm.graph.nodes
|
||||
if n.op == "call_function" and n.target is helion_kernel_wrapper_mutation
|
||||
]
|
||||
assert len(hop_nodes) == 1
|
||||
node = hop_nodes[0]
|
||||
|
||||
assert node.kwargs["constant_args"]["scale"] == scale
|
||||
assert set(node.kwargs["tensor_args"]) == {"x", "y"}
|
||||
|
||||
specs = node.kwargs["output_spec"]["leaf_specs"]
|
||||
tensor_specs = [s for s in specs if s["type"] == "tensor"]
|
||||
scalar_specs = [s for s in specs if s["type"] == "scalar"]
|
||||
assert len(tensor_specs) == 2
|
||||
assert len(scalar_specs) == 1
|
||||
|
||||
for spec in tensor_specs:
|
||||
assert spec["dtype"] == input_x.dtype
|
||||
|
||||
assert scalar_specs[0]["scalar_value"] == 42
|
||||
|
||||
for val in node.meta["val"]:
|
||||
assert all(isinstance(s, torch.SymInt) for s in val.shape)
|
||||
|
||||
# Both out_x and out_y are empty_like(x), so output shapes == input shape
|
||||
input_node = next(n for n in gm.graph.nodes if n.op == "placeholder")
|
||||
input_shape = input_node.meta["val"].shape
|
||||
for val in node.meta["val"]:
|
||||
assert len(val.shape) == len(input_shape)
|
||||
for out_s, in_s in zip(val.shape, input_shape):
|
||||
assert out_s == in_s
|
||||
|
||||
@pytest.mark.skip(reason="SymInt proxy tracking issue with PyTorch 2.11+")
|
||||
def test_pattern_matcher_replaces_with_helion_hop(self):
|
||||
def raw_silu_mul(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
M, N = x.size()
|
||||
out = torch.empty_like(x)
|
||||
for tile_m, tile_n in hl.tile([M, N]):
|
||||
out[tile_m, tile_n] = (
|
||||
torch.nn.functional.silu(x[tile_m, tile_n]) * y[tile_m, tile_n]
|
||||
)
|
||||
return out
|
||||
|
||||
with _helion_mock_context():
|
||||
wrapper = HelionKernelWrapper(
|
||||
raw_kernel_func=raw_silu_mul,
|
||||
op_name="test_pm_silu_mul",
|
||||
fake_impl=lambda *a, **kw: None,
|
||||
config_picker=lambda args, keys: CaseKey.default(),
|
||||
)
|
||||
|
||||
def pattern(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return torch.nn.functional.silu(x) * y
|
||||
|
||||
def replacement(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return wrapper(x, y)
|
||||
|
||||
inputs = [torch.randn(8, 16), torch.randn(8, 16)]
|
||||
|
||||
pm_pass = PatternMatcherPass(pass_name="test_helion_replacement")
|
||||
register_replacement(pattern, replacement, inputs, fwd_only, pm_pass)
|
||||
|
||||
def model(x, y):
|
||||
return torch.nn.functional.silu(x) * y
|
||||
|
||||
decompositions = select_decomp_table()
|
||||
input_x = torch.randn(8, 16)
|
||||
input_y = torch.randn(8, 16)
|
||||
gm = make_fx(model, decompositions, tracing_mode="symbolic")(
|
||||
input_x, input_y
|
||||
)
|
||||
|
||||
def count_hop_nodes(graph):
|
||||
return sum(
|
||||
1
|
||||
for n in graph.nodes
|
||||
if n.op == "call_function"
|
||||
and n.target is helion_kernel_wrapper_mutation
|
||||
)
|
||||
|
||||
assert count_hop_nodes(gm.graph) == 0
|
||||
|
||||
match_count = pm_pass.apply(gm.graph)
|
||||
gm.graph.lint()
|
||||
gm.recompile()
|
||||
|
||||
assert match_count == 1
|
||||
assert count_hop_nodes(gm.graph) == 1
|
||||
|
||||
hop_node = next(
|
||||
n
|
||||
for n in gm.graph.nodes
|
||||
if n.op == "call_function"
|
||||
and n.target is helion_kernel_wrapper_mutation
|
||||
)
|
||||
|
||||
# raw_silu_mul returns empty_like(x), so output shape == input shape
|
||||
for val in hop_node.meta["val"]:
|
||||
assert all(isinstance(s, torch.SymInt) for s in val.shape)
|
||||
|
||||
input_node = next(n for n in gm.graph.nodes if n.op == "placeholder")
|
||||
input_shape = input_node.meta["val"].shape
|
||||
output_shape = hop_node.meta["val"][0].shape
|
||||
assert len(output_shape) == len(input_shape)
|
||||
for out_s, in_s in zip(output_shape, input_shape):
|
||||
assert out_s == in_s
|
||||
@@ -1,232 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the per_token_group_fp8_quant helion kernel.
|
||||
|
||||
Run `pytest tests/kernels/helion/test_per_token_group_fp8_quant.py`.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
from tests.kernels.helion.utils import skip_if_platform_unsupported
|
||||
from tests.kernels.quant_utils import FP8_DTYPE
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.ops.per_token_group_fp8_quant import (
|
||||
_pick_cache,
|
||||
baseline,
|
||||
per_token_group_fp8_quant,
|
||||
pick_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
pytest.skip(
|
||||
"Helion is not installed. Install with: pip install vllm[helion]",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def _generate_fake_input(
|
||||
num_tokens: int, hidden_size: int, group_size: int
|
||||
) -> tuple[Any, ...]:
|
||||
with FakeTensorMode():
|
||||
input = torch.randn(
|
||||
(num_tokens, hidden_size), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
output_q = torch.empty(input.shape, device=input.device, dtype=FP8_DTYPE)
|
||||
output_s = torch.empty(
|
||||
(num_tokens, hidden_size // group_size),
|
||||
device=input.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
use_ue8m0 = False
|
||||
column_major = False
|
||||
fp8_min, fp8_max = get_fp8_min_max()
|
||||
eps = 1e-10
|
||||
args = (
|
||||
input,
|
||||
output_q,
|
||||
output_s,
|
||||
group_size,
|
||||
eps,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
use_ue8m0,
|
||||
column_major,
|
||||
)
|
||||
return args
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_config_manager_singleton():
|
||||
ConfigManager.reset_instance()
|
||||
ConfigManager()
|
||||
yield
|
||||
ConfigManager.reset_instance()
|
||||
|
||||
|
||||
class TestPerTokenGroupFp8QuantConfigPicker:
|
||||
def setup_method(self):
|
||||
_pick_cache.clear()
|
||||
|
||||
def test_config_picker_exact_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"hidden_size": 4096, "group_size": 128, "num_tokens": 16}
|
||||
)
|
||||
|
||||
def test_config_picker_closest_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 32}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(20, 3000, 70)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"hidden_size": 2048, "group_size": 64, "num_tokens": 32}
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key is None
|
||||
|
||||
def test_config_picker_fallback_to_largest(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 32}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(64, 8192, 256)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"hidden_size": 4096, "group_size": 128, "num_tokens": 32}
|
||||
)
|
||||
|
||||
|
||||
class TestPerTokenGroupFp8QuantCorrectness:
|
||||
@pytest.mark.parametrize(
|
||||
"shape", [(31, 128), (32, 128), (63, 256), (64, 256), (16, 512), (2048, 5120)]
|
||||
)
|
||||
@pytest.mark.parametrize("column_major", [False, True])
|
||||
@pytest.mark.parametrize("tma_aligned", [False, True])
|
||||
@pytest.mark.parametrize("scale_ue8m0", [False, True])
|
||||
@pytest.mark.parametrize("group_size", [64, 128])
|
||||
def test_per_token_group_fp8_quant(
|
||||
self,
|
||||
shape,
|
||||
column_major: bool,
|
||||
tma_aligned: bool,
|
||||
scale_ue8m0: bool,
|
||||
group_size: int,
|
||||
):
|
||||
skip_if_platform_unsupported("per_token_group_fp8_quant")
|
||||
|
||||
torch.manual_seed(42)
|
||||
num_tokens, hidden_size = shape
|
||||
fp8_min, fp8_max = get_fp8_min_max()
|
||||
eps = 1e-10
|
||||
input = (
|
||||
torch.randn((num_tokens, hidden_size), device="cuda", dtype=torch.bfloat16)
|
||||
* 8
|
||||
)
|
||||
ref_q = torch.empty(input.shape, device=input.device, dtype=FP8_DTYPE)
|
||||
ops_q = ref_q.clone()
|
||||
|
||||
groups_per_row = hidden_size // group_size
|
||||
if column_major:
|
||||
if tma_aligned:
|
||||
tma_alignment = 4
|
||||
tma_aligned_m = (
|
||||
(num_tokens + tma_alignment - 1) // tma_alignment * tma_alignment
|
||||
)
|
||||
shape = (num_tokens, groups_per_row)
|
||||
stride = (1, tma_aligned_m)
|
||||
ref_s = torch.empty_strided(
|
||||
shape, stride, device=input.device, dtype=torch.float32
|
||||
)
|
||||
else:
|
||||
ref_s = torch.empty(
|
||||
(groups_per_row, num_tokens),
|
||||
device=input.device,
|
||||
dtype=torch.float32,
|
||||
).transpose(0, 1)
|
||||
else:
|
||||
ref_s = torch.empty(
|
||||
(num_tokens, groups_per_row), device=input.device, dtype=torch.float32
|
||||
)
|
||||
|
||||
ops_s = ref_s.clone()
|
||||
|
||||
baseline(
|
||||
input,
|
||||
ref_q,
|
||||
ref_s,
|
||||
group_size,
|
||||
eps,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
scale_ue8m0,
|
||||
column_major,
|
||||
tma_aligned,
|
||||
)
|
||||
per_token_group_fp8_quant(
|
||||
input,
|
||||
ops_q,
|
||||
ops_s,
|
||||
group_size,
|
||||
eps,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
scale_ue8m0,
|
||||
column_major,
|
||||
tma_aligned,
|
||||
)
|
||||
|
||||
assert torch.allclose(ref_s, ops_s)
|
||||
# allow 1 ULP difference
|
||||
assert (
|
||||
ref_q.view(torch.uint8).to(torch.int16)
|
||||
- ops_q.view(torch.uint8).to(torch.int16)
|
||||
).abs().max() <= 1
|
||||
|
||||
|
||||
class TestPerTokenGroupFp8QuantIntegration:
|
||||
def test_kernel_registration_integration(self):
|
||||
from vllm.kernels.helion.register import get_registered_kernels
|
||||
|
||||
registered_kernels = get_registered_kernels()
|
||||
assert "per_token_group_fp8_quant" in registered_kernels
|
||||
|
||||
kernel_wrapper = registered_kernels["per_token_group_fp8_quant"]
|
||||
assert kernel_wrapper.op_name == "per_token_group_fp8_quant"
|
||||
assert kernel_wrapper._config_picker is not None
|
||||
assert kernel_wrapper._mutates_args == ["output_q", "output_s"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,196 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the rms_norm_dynamic_per_token_quant helion kernel.
|
||||
|
||||
Run `pytest tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py`.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
from tests.kernels.helion.utils import skip_if_platform_unsupported
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.ops.rms_norm_dynamic_per_token_quant import (
|
||||
_pick_cache,
|
||||
baseline,
|
||||
pick_config,
|
||||
rms_norm_dynamic_per_token_quant,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_helion
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not has_helion():
|
||||
pytest.skip(
|
||||
"Helion is not installed. Install with: pip install vllm[helion]",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def _generate_fake_input(num_tokens: int, hidden_size: int) -> tuple[Any, ...]:
|
||||
with FakeTensorMode():
|
||||
input = torch.randn(
|
||||
num_tokens, hidden_size, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
result = torch.empty(
|
||||
input.shape, device=input.device, dtype=current_platform.fp8_dtype()
|
||||
)
|
||||
scale = torch.empty((num_tokens, 1), device=input.device, dtype=torch.float32)
|
||||
scale_ub = torch.mean(input).to(torch.float32)
|
||||
residual = torch.randn_like(input)
|
||||
weight = torch.normal(
|
||||
mean=1.0,
|
||||
std=1.0,
|
||||
size=(hidden_size,),
|
||||
dtype=input.dtype,
|
||||
device=input.device,
|
||||
)
|
||||
epsilon = 1e-6
|
||||
args = (result, input, weight, scale, epsilon, scale_ub, residual)
|
||||
return args
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_config_manager_singleton():
|
||||
ConfigManager.reset_instance()
|
||||
ConfigManager()
|
||||
yield
|
||||
ConfigManager.reset_instance()
|
||||
|
||||
|
||||
class TestRmsNormDynamicPerTokenQuantConfigPicker:
|
||||
def setup_method(self):
|
||||
_pick_cache.clear()
|
||||
|
||||
def test_config_picker_exact_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(16, 4096)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey({"hidden_size": 4096, "num_tokens": 16})
|
||||
|
||||
def test_config_picker_closest_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 2048, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "num_tokens": 32}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(20, 3000)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key is None
|
||||
|
||||
def test_config_picker_fallback_to_largest(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "num_tokens": 16}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(32, 8192)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey({"hidden_size": 4096, "num_tokens": 16})
|
||||
|
||||
|
||||
DTYPES = [torch.bfloat16, torch.float]
|
||||
QUANT_DTYPES = [torch.int8, current_platform.fp8_dtype()]
|
||||
VEC_HIDDEN_SIZES = [1024, 1025, 1027, 1029]
|
||||
# Avoid combinatorial explosion with full Cartesian product
|
||||
NUM_TOKENS_HIDDEN_SIZES = [
|
||||
*[(1, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5120, 5137]],
|
||||
*[(2048, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5137]],
|
||||
*[(4096, i) for i in [1, 64, 5137]],
|
||||
]
|
||||
|
||||
ADD_RESIDUAL = [False, True]
|
||||
SCALE_UBS = [True, False]
|
||||
SEEDS = [0]
|
||||
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
class TestRmsNormDynamicPerTokenQuantCorrectness:
|
||||
@pytest.mark.parametrize("num_tokens, hidden_size", NUM_TOKENS_HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
|
||||
@pytest.mark.parametrize("has_scale_ub", SCALE_UBS)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("quant_dtype", QUANT_DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
def test_rms_norm_dynamic_per_token_quant(
|
||||
self,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
add_residual: bool,
|
||||
has_scale_ub: bool,
|
||||
dtype: torch.dtype,
|
||||
quant_dtype: torch.dtype,
|
||||
seed: int,
|
||||
) -> None:
|
||||
skip_if_platform_unsupported("rms_norm_dynamic_per_token_quant")
|
||||
|
||||
set_random_seed(seed)
|
||||
|
||||
if has_scale_ub and quant_dtype != current_platform.fp8_dtype():
|
||||
# skip
|
||||
return
|
||||
|
||||
scale = 1 / (hidden_size)
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device="cuda") * scale
|
||||
weight = torch.normal(
|
||||
mean=1.0, std=1.0, size=(hidden_size,), dtype=dtype, device=x.device
|
||||
)
|
||||
residual = torch.randn_like(x) * scale if add_residual else None
|
||||
scale_ub = (
|
||||
torch.mean(x).to(dtype=torch.float32, device="cuda")
|
||||
if has_scale_ub
|
||||
else None
|
||||
)
|
||||
|
||||
ref_out = torch.empty(x.shape, device=x.device, dtype=quant_dtype)
|
||||
ref_scales = torch.empty((x.shape[0], 1), device=x.device, dtype=torch.float32)
|
||||
ref_residual = residual.clone() if residual is not None else None
|
||||
baseline(ref_out, x, weight, ref_scales, EPS, scale_ub, ref_residual)
|
||||
|
||||
ops_out = torch.empty(x.shape, device=x.device, dtype=quant_dtype)
|
||||
ops_scales = torch.empty((x.shape[0], 1), device=x.device, dtype=torch.float32)
|
||||
ops_residual = residual.clone() if residual is not None else None
|
||||
rms_norm_dynamic_per_token_quant(
|
||||
ops_out, x, weight, ops_scales, EPS, scale_ub, ops_residual
|
||||
)
|
||||
|
||||
torch.testing.assert_close(ref_scales, ops_scales)
|
||||
# allow 1 ULP difference
|
||||
assert (
|
||||
ref_out.view(torch.uint8).to(torch.int16)
|
||||
- ops_out.view(torch.uint8).to(torch.int16)
|
||||
).abs().max() <= 1
|
||||
|
||||
if add_residual:
|
||||
torch.testing.assert_close(ref_residual, ops_residual)
|
||||
|
||||
|
||||
class TestRmsNormDynamicPerTokenQuantIntegration:
|
||||
def test_kernel_registration_integration(self):
|
||||
from vllm.kernels.helion.register import get_registered_kernels
|
||||
|
||||
registered_kernels = get_registered_kernels()
|
||||
assert "rms_norm_dynamic_per_token_quant" in registered_kernels
|
||||
|
||||
kernel_wrapper = registered_kernels["rms_norm_dynamic_per_token_quant"]
|
||||
assert kernel_wrapper.op_name == "rms_norm_dynamic_per_token_quant"
|
||||
assert kernel_wrapper._config_picker is not None
|
||||
assert kernel_wrapper._mutates_args == ["result", "scale", "residual"]
|
||||
@@ -1,300 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the rms_norm_per_block_quant helion kernel.
|
||||
|
||||
Run `pytest tests/kernels/helion/test_rms_norm_per_block_quant.py`.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
from tests.kernels.helion.utils import skip_if_platform_unsupported
|
||||
from tests.kernels.quant_utils import FP8_DTYPE
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.ops.rms_norm_per_block_quant import (
|
||||
_pick_cache,
|
||||
baseline,
|
||||
pick_config,
|
||||
rms_norm_per_block_quant,
|
||||
)
|
||||
from vllm.utils.import_utils import has_helion
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not has_helion():
|
||||
pytest.skip(
|
||||
"Helion is not installed. Install with: pip install vllm[helion]",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def _generate_fake_input(
|
||||
num_tokens: int, hidden_size: int, group_size: int
|
||||
) -> tuple[Any, ...]:
|
||||
with FakeTensorMode():
|
||||
input = torch.randn(
|
||||
(num_tokens, hidden_size), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
result = torch.empty(input.shape, device=input.device, dtype=FP8_DTYPE)
|
||||
scale = torch.empty(
|
||||
(num_tokens, hidden_size // group_size),
|
||||
device=input.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
scale_ub = torch.mean(input).to(scale.dtype)
|
||||
residual = torch.randn_like(input)
|
||||
weight = torch.normal(
|
||||
mean=1.0,
|
||||
std=1.0,
|
||||
size=(hidden_size,),
|
||||
dtype=input.dtype,
|
||||
device=input.device,
|
||||
)
|
||||
epsilon = 1e-6
|
||||
args = (
|
||||
result,
|
||||
input,
|
||||
weight,
|
||||
scale,
|
||||
epsilon,
|
||||
scale_ub,
|
||||
residual,
|
||||
group_size,
|
||||
False,
|
||||
)
|
||||
return args
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_config_manager_singleton():
|
||||
ConfigManager.reset_instance()
|
||||
ConfigManager()
|
||||
yield
|
||||
ConfigManager.reset_instance()
|
||||
|
||||
|
||||
class TestRmsNormPerBlockQuantConfigPicker:
|
||||
def setup_method(self):
|
||||
_pick_cache.clear()
|
||||
|
||||
def test_config_picker_exact_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"hidden_size": 4096, "group_size": 128, "num_tokens": 16}
|
||||
)
|
||||
|
||||
def test_config_picker_closest_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 32}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(20, 3000, 70)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"hidden_size": 2048, "group_size": 64, "num_tokens": 32}
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key is None
|
||||
|
||||
def test_config_picker_fallback_to_largest(self):
|
||||
config_keys = [
|
||||
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 2048, "group_size": 128, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"hidden_size": 4096, "group_size": 128, "num_tokens": 32}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(64, 8192, 256)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"hidden_size": 4096, "group_size": 128, "num_tokens": 32}
|
||||
)
|
||||
|
||||
def test_b200_configs_disable_reduction_warp_specialization(self):
|
||||
config_set = ConfigManager.get_instance().load_config_set(
|
||||
"rms_norm_per_block_quant"
|
||||
)
|
||||
configs = config_set.to_dict()["nvidia_b200"].values()
|
||||
|
||||
# TODO: Remove once the Triton pin includes
|
||||
# https://github.com/triton-lang/triton/pull/9716. Tracked by
|
||||
# https://github.com/triton-lang/triton/issues/10901.
|
||||
assert all(
|
||||
config["range_warp_specializes"][1] is not True for config in configs
|
||||
)
|
||||
|
||||
|
||||
DTYPES = [torch.bfloat16, torch.float]
|
||||
QUANT_DTYPES = [torch.int8, FP8_DTYPE]
|
||||
VEC_HIDDEN_SIZES = [64, 1024]
|
||||
# Avoid combinatorial explosion with full Cartesian product
|
||||
NUM_TOKENS_HIDDEN_SIZES = [
|
||||
*[(1, i) for i in [64, 128, 1024, 5120]],
|
||||
*[(2048, i) for i in [64, 1024]],
|
||||
*[(4096, i) for i in [64]],
|
||||
]
|
||||
|
||||
ADD_RESIDUAL = [False, True]
|
||||
SCALE_UBS = [True, False]
|
||||
GROUP_SIZES = [64, 128]
|
||||
TMA_ALIGNMENTS = [0, 4]
|
||||
SEEDS = [0]
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
class TestRmsNormPerBlockQuantCorrectness:
|
||||
@pytest.mark.parametrize("num_tokens, hidden_size", NUM_TOKENS_HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
|
||||
@pytest.mark.parametrize("has_scale_ub", SCALE_UBS)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("quant_dtype", QUANT_DTYPES)
|
||||
@pytest.mark.parametrize("is_scale_transposed", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"group_size, tma_alignment",
|
||||
[*itertools.product(GROUP_SIZES, TMA_ALIGNMENTS)],
|
||||
)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
def test_rms_norm_per_block_quant(
|
||||
self,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
add_residual: bool,
|
||||
has_scale_ub: bool,
|
||||
dtype: torch.dtype,
|
||||
quant_dtype: torch.dtype,
|
||||
is_scale_transposed: bool,
|
||||
group_size: int,
|
||||
tma_alignment: int,
|
||||
seed: int,
|
||||
) -> None:
|
||||
skip_if_platform_unsupported("rms_norm_per_block_quant")
|
||||
|
||||
set_random_seed(seed)
|
||||
|
||||
if hidden_size % group_size != 0:
|
||||
# skip
|
||||
return
|
||||
|
||||
if tma_alignment != 0 and hidden_size // group_size % tma_alignment == 0:
|
||||
# Skip tests where TMA alignment doesn't create extra padding to save time
|
||||
return
|
||||
|
||||
if has_scale_ub and quant_dtype != FP8_DTYPE:
|
||||
# skip
|
||||
return
|
||||
|
||||
scale = 1 / (hidden_size)
|
||||
input = torch.randn(num_tokens, hidden_size, dtype=dtype, device="cuda") * scale
|
||||
weight = torch.normal(
|
||||
mean=1.0, std=1.0, size=(hidden_size,), dtype=dtype, device=input.device
|
||||
)
|
||||
residual = torch.randn_like(input) * scale if add_residual else None
|
||||
scale_ub = (
|
||||
torch.mean(input).to(dtype=torch.float32, device="cuda")
|
||||
if has_scale_ub
|
||||
else None
|
||||
)
|
||||
groups_per_row = hidden_size // group_size
|
||||
|
||||
ref_residual = residual.clone() if residual is not None else None
|
||||
ops_residual = residual.clone() if residual is not None else None
|
||||
ref_out = torch.empty(input.shape, device=input.device, dtype=quant_dtype)
|
||||
ops_out = ref_out.clone()
|
||||
|
||||
if is_scale_transposed:
|
||||
if tma_alignment == 0:
|
||||
ref_scales = torch.empty(
|
||||
(groups_per_row, num_tokens),
|
||||
device=input.device,
|
||||
dtype=torch.float32,
|
||||
).transpose(0, 1)
|
||||
else:
|
||||
tma_aligned_m = (
|
||||
(num_tokens + tma_alignment - 1) // tma_alignment * tma_alignment
|
||||
)
|
||||
shape = (num_tokens, groups_per_row)
|
||||
stride = (1, tma_aligned_m)
|
||||
ref_scales = torch.empty_strided(
|
||||
shape, stride, device=input.device, dtype=torch.float32
|
||||
)
|
||||
else:
|
||||
ref_scales = torch.empty(
|
||||
(num_tokens, groups_per_row), device=input.device, dtype=torch.float32
|
||||
)
|
||||
|
||||
ops_scales = ref_scales.clone()
|
||||
|
||||
baseline(
|
||||
ref_out,
|
||||
input,
|
||||
weight,
|
||||
ref_scales,
|
||||
EPS,
|
||||
scale_ub,
|
||||
ref_residual,
|
||||
group_size,
|
||||
is_scale_transposed,
|
||||
)
|
||||
ref_scales = ref_scales.contiguous()
|
||||
|
||||
rms_norm_per_block_quant(
|
||||
ops_out,
|
||||
input,
|
||||
weight,
|
||||
ops_scales,
|
||||
EPS,
|
||||
scale_ub,
|
||||
ops_residual,
|
||||
group_size,
|
||||
is_scale_transposed,
|
||||
)
|
||||
ops_scales = ops_scales.contiguous()
|
||||
|
||||
torch.testing.assert_close(ref_scales, ops_scales)
|
||||
# allow 1 ULP difference
|
||||
assert (
|
||||
ref_out.view(torch.uint8).to(torch.int16)
|
||||
- ops_out.view(torch.uint8).to(torch.int16)
|
||||
).abs().max() <= 1
|
||||
|
||||
if add_residual:
|
||||
torch.testing.assert_close(ref_residual, ops_residual)
|
||||
|
||||
|
||||
class TestRmsNormPerBlockQuantIntegration:
|
||||
def test_kernel_registration_integration(self):
|
||||
from vllm.kernels.helion.register import get_registered_kernels
|
||||
|
||||
registered_kernels = get_registered_kernels()
|
||||
assert "rms_norm_per_block_quant" in registered_kernels
|
||||
|
||||
kernel_wrapper = registered_kernels["rms_norm_per_block_quant"]
|
||||
assert kernel_wrapper.op_name == "rms_norm_per_block_quant"
|
||||
assert kernel_wrapper._config_picker is not None
|
||||
assert kernel_wrapper._mutates_args == ["result", "scale", "residual"]
|
||||
@@ -1,213 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the silu_and_mul_per_block_quant helion kernel
|
||||
Run `pytest tests/kernels/helion/test_silu_and_mul_per_block_quant.py`.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
from tests.kernels.helion.utils import skip_if_platform_unsupported
|
||||
from tests.kernels.quant_utils import FP8_DTYPE
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.ops.silu_and_mul_per_block_quant import (
|
||||
_pick_cache,
|
||||
baseline,
|
||||
pick_config,
|
||||
silu_and_mul_per_block_quant,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_helion
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not has_helion():
|
||||
pytest.skip(
|
||||
"Helion is not installed. Install with: pip install vllm[helion]",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def _generate_fake_input(
|
||||
num_tokens: int, intermediate_size: int, group_size: int
|
||||
) -> tuple[Any, ...]:
|
||||
with FakeTensorMode():
|
||||
in_dtype: torch.dtype = torch.bfloat16
|
||||
out_dtype: torch.dtype = current_platform.fp8_dtype()
|
||||
scale_dtype: torch.dtype = torch.float32
|
||||
input = torch.randn(
|
||||
num_tokens, 2 * intermediate_size, device="cuda", dtype=in_dtype
|
||||
)
|
||||
result = torch.empty(
|
||||
num_tokens, intermediate_size, device=input.device, dtype=out_dtype
|
||||
)
|
||||
scale = torch.empty(
|
||||
(num_tokens, intermediate_size // group_size),
|
||||
device=input.device,
|
||||
dtype=scale_dtype,
|
||||
)
|
||||
scale_ub = torch.mean(input).to(scale_dtype)
|
||||
args = (
|
||||
result,
|
||||
input,
|
||||
scale,
|
||||
group_size,
|
||||
scale_ub,
|
||||
False,
|
||||
)
|
||||
return args
|
||||
|
||||
|
||||
class TestSiluAndMulPerBlockQuantConfigPicker:
|
||||
def setup_method(self):
|
||||
_pick_cache.clear()
|
||||
|
||||
def test_config_picker_exact_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"intermediate_size": 2048, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"intermediate_size": 4096, "group_size": 128, "num_tokens": 16}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"intermediate_size": 4096, "group_size": 128, "num_tokens": 16}
|
||||
)
|
||||
|
||||
def test_config_picker_closest_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"intermediate_size": 2048, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"intermediate_size": 2048, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"intermediate_size": 2048, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"intermediate_size": 2048, "group_size": 128, "num_tokens": 32}),
|
||||
CaseKey({"intermediate_size": 4096, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"intermediate_size": 4096, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"intermediate_size": 4096, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"intermediate_size": 4096, "group_size": 128, "num_tokens": 32}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(20, 3000, 70)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"intermediate_size": 2048, "group_size": 64, "num_tokens": 32}
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key is None
|
||||
|
||||
def test_config_picker_fallback_to_largest(self):
|
||||
config_keys = [
|
||||
CaseKey({"intermediate_size": 2048, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"intermediate_size": 2048, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"intermediate_size": 2048, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"intermediate_size": 2048, "group_size": 128, "num_tokens": 32}),
|
||||
CaseKey({"intermediate_size": 4096, "group_size": 64, "num_tokens": 16}),
|
||||
CaseKey({"intermediate_size": 4096, "group_size": 64, "num_tokens": 32}),
|
||||
CaseKey({"intermediate_size": 4096, "group_size": 128, "num_tokens": 16}),
|
||||
CaseKey({"intermediate_size": 4096, "group_size": 128, "num_tokens": 32}),
|
||||
]
|
||||
|
||||
args = _generate_fake_input(64, 8192, 256)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
assert selected_key == CaseKey(
|
||||
{"intermediate_size": 4096, "group_size": 128, "num_tokens": 32}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_config_manager_singleton():
|
||||
ConfigManager.reset_instance()
|
||||
ConfigManager()
|
||||
yield
|
||||
ConfigManager.reset_instance()
|
||||
|
||||
|
||||
class TestSiluAndMulPerBlockQuantCorrectness:
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 4096])
|
||||
@pytest.mark.parametrize("hidden_size", [1024, 2048, 5120])
|
||||
@pytest.mark.parametrize("group_size", [64, 128])
|
||||
@pytest.mark.parametrize("is_scale_transposed", [False, True])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("quant_dtype", [current_platform.fp8_dtype(), torch.int8])
|
||||
@pytest.mark.parametrize("has_scale_ub", [True, False])
|
||||
@pytest.mark.parametrize("seed", [0])
|
||||
def test_silu_and_mul_per_block_quant(
|
||||
self,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
group_size: int,
|
||||
is_scale_transposed: bool,
|
||||
dtype: torch.dtype,
|
||||
quant_dtype: torch.dtype,
|
||||
has_scale_ub: bool,
|
||||
seed: int,
|
||||
) -> None:
|
||||
skip_if_platform_unsupported("silu_and_mul_per_block_quant")
|
||||
set_random_seed(seed)
|
||||
|
||||
if hidden_size % group_size != 0:
|
||||
return
|
||||
|
||||
if has_scale_ub and quant_dtype != FP8_DTYPE:
|
||||
# skip
|
||||
return
|
||||
|
||||
scale = 1 / hidden_size
|
||||
x = torch.randn(num_tokens, 2 * hidden_size, dtype=dtype, device="cuda") * scale
|
||||
|
||||
if has_scale_ub:
|
||||
act = torch.nn.functional.silu(x[:, :hidden_size]) * x[:, hidden_size:]
|
||||
act_abs = act.abs().float()
|
||||
scale_ub = 0.5 * (act_abs.mean() + act_abs.amax())
|
||||
else:
|
||||
scale_ub = None
|
||||
|
||||
ref_out = torch.empty(num_tokens, hidden_size, device="cuda", dtype=quant_dtype)
|
||||
|
||||
if is_scale_transposed:
|
||||
ref_scales = torch.empty(
|
||||
(hidden_size // group_size, x.shape[0]),
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
).t()
|
||||
else:
|
||||
ref_scales = torch.empty(
|
||||
(x.shape[0], hidden_size // group_size),
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
ops_out = ref_out.clone()
|
||||
ops_scales = ref_scales.clone()
|
||||
|
||||
baseline(ref_out, x, ref_scales, group_size, scale_ub, is_scale_transposed)
|
||||
silu_and_mul_per_block_quant(
|
||||
ops_out, x, ops_scales, group_size, scale_ub, is_scale_transposed
|
||||
)
|
||||
|
||||
torch.testing.assert_close(ref_scales, ops_scales)
|
||||
# allow 1 ULP difference
|
||||
assert (
|
||||
ref_out.view(torch.uint8).to(torch.int16)
|
||||
- ops_out.view(torch.uint8).to(torch.int16)
|
||||
).abs().max() <= 1
|
||||
|
||||
|
||||
class TestSiluAndMulPerBlockQuantIntegration:
|
||||
def test_kernel_registration_integration(self):
|
||||
from vllm.kernels.helion.register import get_registered_kernels
|
||||
|
||||
registered_kernels = get_registered_kernels()
|
||||
assert "silu_and_mul_per_block_quant" in registered_kernels
|
||||
|
||||
kernel_wrapper = registered_kernels["silu_and_mul_per_block_quant"]
|
||||
assert kernel_wrapper.op_name == "silu_and_mul_per_block_quant"
|
||||
assert kernel_wrapper._config_picker is not None
|
||||
assert kernel_wrapper._mutates_args == ["out", "scales"]
|
||||
@@ -1,365 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
pytest.skip(
|
||||
"Helion is not installed. Install with: pip install vllm[helion]",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.ops.silu_mul_fp8 import (
|
||||
_pick_cache,
|
||||
pick_silu_mul_fp8_config,
|
||||
silu_mul_fp8,
|
||||
silu_mul_fp8_baseline,
|
||||
)
|
||||
|
||||
|
||||
def skip_if_platform_unsupported():
|
||||
try:
|
||||
from vllm.kernels.helion.utils import get_canonical_gpu_name
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
|
||||
platform = get_canonical_gpu_name()
|
||||
|
||||
try:
|
||||
config_manager = ConfigManager.get_instance()
|
||||
except RuntimeError:
|
||||
config_manager = ConfigManager()
|
||||
|
||||
configs = config_manager.get_platform_configs("silu_mul_fp8", platform)
|
||||
if len(configs) == 0:
|
||||
pytest.skip("Current GPU platform not supported for silu_mul_fp8 kernel")
|
||||
|
||||
except (ImportError, RuntimeError, KeyError):
|
||||
pytest.skip("Error detecting platform support for silu_mul_fp8 kernel")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_config_manager_singleton():
|
||||
ConfigManager.reset_instance()
|
||||
ConfigManager()
|
||||
yield
|
||||
ConfigManager.reset_instance()
|
||||
|
||||
|
||||
class TestSiluMulFp8ConfigPicker:
|
||||
def setup_method(self):
|
||||
_pick_cache.clear()
|
||||
|
||||
def test_config_picker_exact_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"intermediate": 2048, "numtokens": 256}),
|
||||
CaseKey({"intermediate": 4096, "numtokens": 256}),
|
||||
]
|
||||
|
||||
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
args = (input_tensor, scale)
|
||||
|
||||
selected_key = pick_silu_mul_fp8_config(args, config_keys)
|
||||
assert selected_key == CaseKey({"intermediate": 2048, "numtokens": 256})
|
||||
|
||||
def test_config_picker_closest_match(self):
|
||||
config_keys = [
|
||||
CaseKey({"intermediate": 2048, "numtokens": 256}),
|
||||
CaseKey({"intermediate": 4096, "numtokens": 256}),
|
||||
]
|
||||
input_tensor = torch.randn(32, 7000, dtype=torch.bfloat16, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
args = (input_tensor, scale)
|
||||
|
||||
selected_key = pick_silu_mul_fp8_config(args, config_keys)
|
||||
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 256})
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
args = (input_tensor, scale)
|
||||
|
||||
selected_key = pick_silu_mul_fp8_config(args, config_keys)
|
||||
assert selected_key is None
|
||||
|
||||
@pytest.mark.parametrize("intermediate_size", [2048, 4096, 5120])
|
||||
def test_config_picker_different_sizes(self, intermediate_size):
|
||||
config_keys = [
|
||||
CaseKey({"intermediate": 2048, "numtokens": 256}),
|
||||
CaseKey({"intermediate": 4096, "numtokens": 256}),
|
||||
CaseKey({"intermediate": 5120, "numtokens": 256}),
|
||||
]
|
||||
|
||||
input_tensor = torch.randn(
|
||||
32, 2 * intermediate_size, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
args = (input_tensor, scale)
|
||||
|
||||
selected_key = pick_silu_mul_fp8_config(args, config_keys)
|
||||
assert selected_key == {
|
||||
"intermediate": intermediate_size,
|
||||
"numtokens": 256,
|
||||
}
|
||||
|
||||
def test_config_picker_numtokens_ceiling(self):
|
||||
config_keys = [
|
||||
CaseKey({"intermediate": 4096, "numtokens": 8}),
|
||||
CaseKey({"intermediate": 4096, "numtokens": 32}),
|
||||
CaseKey({"intermediate": 4096, "numtokens": 128}),
|
||||
CaseKey({"intermediate": 4096, "numtokens": 256}),
|
||||
]
|
||||
input_tensor = torch.randn(20, 8192, dtype=torch.bfloat16, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
|
||||
selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys)
|
||||
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 32})
|
||||
|
||||
def test_config_picker_numtokens_exact(self):
|
||||
config_keys = [
|
||||
CaseKey({"intermediate": 4096, "numtokens": 8}),
|
||||
CaseKey({"intermediate": 4096, "numtokens": 32}),
|
||||
CaseKey({"intermediate": 4096, "numtokens": 128}),
|
||||
]
|
||||
input_tensor = torch.randn(32, 8192, dtype=torch.bfloat16, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
|
||||
selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys)
|
||||
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 32})
|
||||
|
||||
def test_config_picker_numtokens_fallback_to_largest(self):
|
||||
config_keys = [
|
||||
CaseKey({"intermediate": 4096, "numtokens": 8}),
|
||||
CaseKey({"intermediate": 4096, "numtokens": 32}),
|
||||
CaseKey({"intermediate": 4096, "numtokens": 128}),
|
||||
]
|
||||
input_tensor = torch.randn(512, 8192, dtype=torch.bfloat16, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
|
||||
selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys)
|
||||
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 128})
|
||||
|
||||
|
||||
class TestSiluMulFp8Correctness:
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
|
||||
@pytest.mark.parametrize("intermediate_size", [2048, 3000, 3500, 4096, 5000])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_silu_mul_fp8_correctness(self, batch_size, intermediate_size, dtype):
|
||||
skip_if_platform_unsupported()
|
||||
|
||||
input_size = 2 * intermediate_size
|
||||
input_tensor = torch.randn(batch_size, input_size, dtype=dtype, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
|
||||
reference_output = silu_mul_fp8_baseline(input_tensor, scale)
|
||||
helion_output = silu_mul_fp8(input_tensor, scale)
|
||||
|
||||
assert helion_output.shape == reference_output.shape
|
||||
assert helion_output.dtype == torch.float8_e4m3fn
|
||||
assert reference_output.dtype == torch.float8_e4m3fn
|
||||
|
||||
ref_f32 = reference_output.to(torch.float32)
|
||||
helion_f32 = helion_output.to(torch.float32)
|
||||
# FP8 E4M3 has limited precision. Values near quantization boundaries
|
||||
# can round differently due to intermediate precision differences.
|
||||
torch.testing.assert_close(
|
||||
helion_f32,
|
||||
ref_f32,
|
||||
atol=0.05,
|
||||
rtol=0.05,
|
||||
msg=f"Mismatch at batch={batch_size}, size={intermediate_size}",
|
||||
)
|
||||
|
||||
def test_silu_mul_fp8_shape_inference(self):
|
||||
skip_if_platform_unsupported()
|
||||
batch_size, input_size = 32, 8192
|
||||
intermediate_size = input_size // 2
|
||||
|
||||
input_tensor = torch.randn(
|
||||
batch_size, input_size, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
|
||||
output = silu_mul_fp8(input_tensor, scale)
|
||||
|
||||
expected_shape = (batch_size, intermediate_size)
|
||||
assert output.shape == expected_shape
|
||||
assert output.dtype == torch.float8_e4m3fn
|
||||
|
||||
def test_silu_mul_fp8_scale_variations(self):
|
||||
skip_if_platform_unsupported()
|
||||
batch_size, input_size = 16, 4096
|
||||
|
||||
input_tensor = torch.randn(
|
||||
batch_size, input_size, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
|
||||
scales = [0.1, 0.5, 1.0, 2.0, 10.0]
|
||||
|
||||
for scale_val in scales:
|
||||
scale = torch.tensor([scale_val], dtype=torch.float32, device="cuda")
|
||||
|
||||
reference_output = silu_mul_fp8_baseline(input_tensor, scale)
|
||||
helion_output = silu_mul_fp8(input_tensor, scale)
|
||||
ref_f32 = reference_output.to(torch.float32)
|
||||
helion_f32 = helion_output.to(torch.float32)
|
||||
|
||||
torch.testing.assert_close(
|
||||
helion_f32,
|
||||
ref_f32,
|
||||
atol=0.05,
|
||||
rtol=0.05,
|
||||
msg=f"Mismatch for scale={scale_val}",
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(1, 4096),
|
||||
(16, 4096),
|
||||
(128, 4096),
|
||||
(1024, 4096),
|
||||
(1, 8192),
|
||||
(16, 8192),
|
||||
(128, 8192),
|
||||
],
|
||||
)
|
||||
def test_silu_mul_fp8_various_shapes(self, shape):
|
||||
skip_if_platform_unsupported()
|
||||
|
||||
input_tensor = torch.randn(*shape, dtype=torch.bfloat16, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
|
||||
reference_output = silu_mul_fp8_baseline(input_tensor, scale)
|
||||
helion_output = silu_mul_fp8(input_tensor, scale)
|
||||
|
||||
assert helion_output.shape == reference_output.shape
|
||||
|
||||
ref_f32 = reference_output.to(torch.float32)
|
||||
helion_f32 = helion_output.to(torch.float32)
|
||||
|
||||
torch.testing.assert_close(
|
||||
helion_f32, ref_f32, atol=0.05, rtol=0.05, msg=f"Mismatch for shape={shape}"
|
||||
)
|
||||
|
||||
|
||||
def silu_mul_fp8_pytorch(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
"""Pure PyTorch reference using F.silu.
|
||||
|
||||
This matches vLLM's SiluAndMul.forward_native exactly:
|
||||
F.silu(x[..., :d]) * x[..., d:]
|
||||
"""
|
||||
d = input.shape[-1] // 2
|
||||
result = F.silu(input[..., :d]) * input[..., d:]
|
||||
return (result.to(torch.float32) / scale).to(torch.float8_e4m3fn)
|
||||
|
||||
|
||||
class TestSiluMulFp8PytorchReference:
|
||||
"""Tests comparing Helion kernel against pure PyTorch implementation.
|
||||
|
||||
Uses tighter tolerance since both use PyTorch's FP8 conversion
|
||||
(same rounding mode), unlike the vLLM C++ baseline which uses
|
||||
NVIDIA's hardware FP8 conversion with different rounding.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 256])
|
||||
@pytest.mark.parametrize("intermediate_size", [1024, 2048, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_silu_mul_fp8_vs_pytorch(self, batch_size, intermediate_size, dtype):
|
||||
skip_if_platform_unsupported()
|
||||
|
||||
input_tensor = torch.randn(
|
||||
batch_size, 2 * intermediate_size, dtype=dtype, device="cuda"
|
||||
)
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
|
||||
pytorch_output = silu_mul_fp8_pytorch(input_tensor, scale)
|
||||
helion_output = silu_mul_fp8(input_tensor, scale)
|
||||
|
||||
assert helion_output.shape == pytorch_output.shape
|
||||
assert helion_output.dtype == torch.float8_e4m3fn
|
||||
|
||||
pytorch_f32 = pytorch_output.to(torch.float32)
|
||||
helion_f32 = helion_output.to(torch.float32)
|
||||
|
||||
# Tolerance accounts for FP8 quantization boundary effects
|
||||
torch.testing.assert_close(
|
||||
helion_f32,
|
||||
pytorch_f32,
|
||||
atol=0.05,
|
||||
rtol=0.05,
|
||||
msg=(
|
||||
f"Mismatch at batch={batch_size}, size={intermediate_size}, "
|
||||
f"dtype={dtype}"
|
||||
),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(1, 2, 4096), # 3D input
|
||||
(2, 4, 2048), # 3D input
|
||||
(1, 1, 1, 8192), # 4D input
|
||||
],
|
||||
)
|
||||
def test_silu_mul_fp8_multidim_vs_pytorch(self, shape):
|
||||
skip_if_platform_unsupported()
|
||||
|
||||
input_tensor = torch.randn(*shape, dtype=torch.bfloat16, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
|
||||
pytorch_output = silu_mul_fp8_pytorch(input_tensor, scale)
|
||||
helion_output = silu_mul_fp8(input_tensor, scale)
|
||||
|
||||
assert helion_output.shape == pytorch_output.shape
|
||||
|
||||
pytorch_f32 = pytorch_output.to(torch.float32)
|
||||
helion_f32 = helion_output.to(torch.float32)
|
||||
|
||||
torch.testing.assert_close(
|
||||
helion_f32,
|
||||
pytorch_f32,
|
||||
atol=0.05,
|
||||
rtol=0.05,
|
||||
msg=f"Mismatch for shape={shape}",
|
||||
)
|
||||
|
||||
|
||||
class TestSiluMulFp8Integration:
|
||||
def test_kernel_registration_integration(self):
|
||||
from vllm.kernels.helion.register import get_registered_kernels
|
||||
|
||||
registered_kernels = get_registered_kernels()
|
||||
assert "silu_mul_fp8" in registered_kernels
|
||||
|
||||
kernel_wrapper = registered_kernels["silu_mul_fp8"]
|
||||
assert kernel_wrapper.op_name == "silu_mul_fp8"
|
||||
assert kernel_wrapper._config_picker is not None
|
||||
|
||||
def test_fake_impl_functionality(self):
|
||||
skip_if_platform_unsupported()
|
||||
from vllm.kernels.helion.register import get_registered_kernels
|
||||
|
||||
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
registered_kernels = get_registered_kernels()
|
||||
kernel_wrapper = registered_kernels["silu_mul_fp8"]
|
||||
fake_impl = kernel_wrapper._fake_impl
|
||||
assert fake_impl is not None
|
||||
|
||||
fake_output = fake_impl(input_tensor, scale)
|
||||
|
||||
expected_shape = (32, 2048)
|
||||
assert fake_output.shape == expected_shape
|
||||
assert fake_output.dtype == torch.float8_e4m3fn
|
||||
assert fake_output.device == input_tensor.device
|
||||
@@ -1,33 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for Helion utility functions."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.kernels.helion.utils import canonicalize_gpu_name
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"driver_reported_name,expected",
|
||||
[
|
||||
("NVIDIA H200", "nvidia_h200"),
|
||||
("NVIDIA A100-SXM4-80GB", "nvidia_a100"),
|
||||
("NVIDIA H100 80GB HBM3", "nvidia_h100"),
|
||||
("NVIDIA H100 PCIe", "nvidia_h100"),
|
||||
("NVIDIA H100 SXM5", "nvidia_h100"),
|
||||
("NVIDIA GeForce RTX 4090", "nvidia_geforce_rtx_4090"),
|
||||
("AMD Instinct MI300X", "amd_instinct_mi300x"),
|
||||
("AMD Instinct MI250X / MI250", "amd_instinct_mi250x_mi250"),
|
||||
("Tesla V100-SXM2-32GB", "tesla_v100"),
|
||||
],
|
||||
)
|
||||
def test_canonicalize_gpu_name(driver_reported_name, expected):
|
||||
"""Test GPU name canonicalization."""
|
||||
assert canonicalize_gpu_name(driver_reported_name) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid_name", ["", " ", "\t", "\n"])
|
||||
def test_canonicalize_gpu_name_rejects_empty(invalid_name):
|
||||
"""Test that empty or whitespace-only names are rejected."""
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
canonicalize_gpu_name(invalid_name)
|
||||
@@ -1,30 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Helion Kernel test utils."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
|
||||
|
||||
def skip_if_platform_unsupported(op_name: str):
|
||||
try:
|
||||
from vllm.kernels.helion.utils import get_canonical_gpu_name
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
|
||||
platform = get_canonical_gpu_name()
|
||||
|
||||
try:
|
||||
config_manager = ConfigManager.get_instance()
|
||||
except RuntimeError:
|
||||
config_manager = ConfigManager()
|
||||
|
||||
configs = config_manager.get_platform_configs(op_name, platform)
|
||||
if len(configs) == 0:
|
||||
pytest.skip(f"Current GPU platform not supported for {op_name} kernel")
|
||||
|
||||
except (ImportError, RuntimeError, KeyError):
|
||||
pytest.skip(f"Error detecting platform support for {op_name} kernel")
|
||||
Reference in New Issue
Block a user