[Kernel][Helion][1/N] Add Helion kernel for fused_qk_norm_rope (#44010)

Signed-off-by: Sean Chen <seachen@redhat.com>
This commit is contained in:
Xiaohong (Sean) Chen
2026-06-29 22:54:15 +08:00
committed by GitHub
parent 0ca39c4f1f
commit 49e28e8e91
4 changed files with 5911 additions and 0 deletions
@@ -0,0 +1,261 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the fused_qk_norm_rope helion kernel
Run `pytest tests/kernels/helion/test_fused_qk_norm_rope.py`.
"""
from typing import Any
import pytest
import torch
from torch._subclasses.fake_tensor import FakeTensorMode
from tests.kernels.helion.utils import skip_if_platform_unsupported
from vllm.benchmarks.lib.utils import default_vllm_config
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.ops.fused_qk_norm_rope import (
_pick_cache,
baseline,
fused_qk_norm_rope,
pick_config,
)
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.utils.import_utils import has_helion
if not has_helion():
pytest.skip(
"Helion is not installed. Install with: pip install vllm[helion]",
allow_module_level=True,
)
@default_vllm_config()
def _generate_fake_input(
num_tokens: int, num_q_heads: int, num_kv_heads: int
) -> tuple[Any, ...]:
with FakeTensorMode():
head_dim = 128
eps = 1e-6
is_neox = True
rotary_ratio = 1.0
device = "cuda"
dtype = torch.bfloat16
total_dim = (num_q_heads + 2 * num_kv_heads) * head_dim
qkv = torch.randn(num_tokens, total_dim, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
q_weight = torch.normal(
mean=1.0,
std=1.0,
size=(head_dim,),
dtype=qkv.dtype,
device=device,
)
k_weight = torch.normal(
mean=1.0,
std=1.0,
size=(head_dim,),
dtype=qkv.dtype,
device=device,
)
rotary_dim = int(head_dim * rotary_ratio)
rope = RotaryEmbedding(
head_size=head_dim,
rotary_dim=rotary_dim,
max_position_embeddings=4096,
base=10000.0,
is_neox_style=is_neox,
dtype=dtype,
).to(device)
args = (
qkv,
num_q_heads,
num_kv_heads,
num_kv_heads,
head_dim,
eps,
q_weight,
k_weight,
rope.cos_sin_cache,
is_neox,
positions.view(-1),
)
return args
@pytest.fixture(autouse=True)
def reset_config_manager_singleton():
ConfigManager.reset_instance()
ConfigManager()
yield
ConfigManager.reset_instance()
class TestFusedQkNormRopeConfigPicker:
def setup_method(self):
_pick_cache.clear()
def test_config_picker_exact_match(self):
config_keys = [
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
]
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}
)
def test_config_picker_closest_match(self):
config_keys = [
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}),
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 16}),
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 32}),
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 16}),
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 32}),
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}),
]
args = _generate_fake_input(20, 3000, 70)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}
)
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
assert selected_key is None
def test_config_picker_fallback_to_largest(self):
config_keys = [
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}),
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 16}),
CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 32}),
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 16}),
CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 32}),
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}),
]
args = _generate_fake_input(64, 8192, 256)
selected_key = pick_config(args, config_keys)
assert selected_key == CaseKey(
{"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}
)
class TestFusedQkNormRopeCorrectness:
@pytest.mark.parametrize(
"num_heads, num_kv_heads, head_dim", [(16, 4, 128), (64, 8, 128)]
)
@pytest.mark.parametrize("num_tokens", [1, 7, 1024, 1025])
@pytest.mark.parametrize("is_neox", [False, True])
@pytest.mark.parametrize("rotary_ratio", [1.0, 0.5, 0.25])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@default_vllm_config()
def test_fused_qk_norm_rope(
self,
num_heads: int,
num_kv_heads: int,
head_dim: int,
num_tokens: int,
is_neox: bool,
rotary_ratio: float,
dtype: torch.dtype,
):
skip_if_platform_unsupported("fused_qk_norm_rope")
torch.manual_seed(42)
eps = 1e-6
device = "cuda"
total_dim = (num_heads + 2 * num_kv_heads) * head_dim
ref_qkv = torch.empty(
num_tokens, total_dim, dtype=dtype, device=device
).uniform_(-0.1, 0.1)
ops_qkv = ref_qkv.clone()
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
q_weight = torch.empty(head_dim, dtype=dtype, device=device).uniform_(0.8, 1.2)
k_weight = torch.empty(head_dim, dtype=dtype, device=device).uniform_(0.8, 1.2)
rotary_dim = int(head_dim * rotary_ratio)
rope = RotaryEmbedding(
head_size=head_dim,
rotary_dim=rotary_dim,
max_position_embeddings=40960,
base=10000.0,
is_neox_style=is_neox,
dtype=dtype,
).to(device)
baseline(
ref_qkv,
num_heads,
num_kv_heads,
num_kv_heads,
head_dim,
eps,
q_weight,
k_weight,
rope.cos_sin_cache,
is_neox,
positions.view(-1),
)
fused_qk_norm_rope(
ops_qkv,
num_heads,
num_kv_heads,
num_kv_heads,
head_dim,
eps,
q_weight,
k_weight,
rope.cos_sin_cache,
is_neox,
positions.view(-1),
)
if dtype == torch.bfloat16:
atol = 5e-2
rtol = 5e-2
else:
atol = 1e-2
rtol = 1e-2
torch.testing.assert_close(
ref_qkv,
ops_qkv,
atol=atol,
rtol=rtol,
)
class TestFusedQkNormRopeIntegration:
def test_kernel_registration_integration(self):
from vllm.kernels.helion.register import get_registered_kernels
registered_kernels = get_registered_kernels()
assert "fused_qk_norm_rope" in registered_kernels
kernel_wrapper = registered_kernels["fused_qk_norm_rope"]
assert kernel_wrapper.op_name == "fused_qk_norm_rope"
assert kernel_wrapper._config_picker is not None
assert kernel_wrapper._mutates_args == ["qkv"]
def test_fake_impl_functionality(self):
skip_if_platform_unsupported("fused_qk_norm_rope")
from vllm.kernels.helion.register import get_registered_kernels
registered_kernels = get_registered_kernels()
kernel_wrapper = registered_kernels["fused_qk_norm_rope"]
fake_impl = kernel_wrapper._fake_impl
args = _generate_fake_input(16, 4096, 128)
assert fake_impl(*args) is None
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,316 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import product
from typing import Any
import torch
from vllm.kernels.helion.case_key import CaseKey
from vllm.logger import init_logger
from vllm.utils.import_utils import has_helion
if not has_helion():
raise ImportError(
"Helion kernel requires helion to be installed. "
"Install it with: pip install helion"
)
import helion
import helion.language as hl
from vllm import ir
from vllm.kernels.helion.register import register_kernel
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
logger = init_logger(__name__)
def _compute_cos_sin_cache(
max_position_embeddings, rotary_dim, device="cuda", dtype=torch.float
):
inv_freq = 1.0 / (
10000
** (torch.arange(0, rotary_dim, 2, device=device, dtype=dtype) / rotary_dim)
)
t = torch.arange(max_position_embeddings, device=device, dtype=dtype)
freqs = torch.einsum("i,j -> ij", t, inv_freq)
cos = freqs.cos()
sin = freqs.sin()
cache = torch.cat((cos, sin), dim=-1)
return cache
def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]:
# TODO(xiaohongchen1991): it is difficult for kernel author to cover
# all input property combination. Currently, dtypes are fixed. We need
# optimization to bucket/skip some combinations
num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
num_heads_pair = [
(16, 8),
(32, 8),
(64, 8),
]
head_dim = 128
in_dtype: torch.dtype = torch.bfloat16
rotary_ratio = 1.0
is_neox = True
eps = 1e-6
device = "cuda"
inputs = {}
for num_tokens, (num_q_heads, num_kv_heads) in product(
num_tokens_list, num_heads_pair
):
total_dim = (num_q_heads + 2 * num_kv_heads) * head_dim
qkv = torch.empty(
num_tokens, total_dim, dtype=in_dtype, device=device
).uniform_(-0.1, 0.1)
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
q_weight = torch.empty(head_dim, dtype=in_dtype, device=device).uniform_(
0.8, 1.2
)
k_weight = torch.empty(head_dim, dtype=in_dtype, device=device).uniform_(
0.8, 1.2
)
rotary_dim = int(head_dim * rotary_ratio)
cos_sin_cache = _compute_cos_sin_cache(40960, rotary_dim)
cos_sin_cache = cos_sin_cache.to(in_dtype)
config_key = CaseKey(
{
"q_heads": num_q_heads,
"kv_heads": num_kv_heads,
"num_tokens": num_tokens,
}
)
inputs[config_key] = (
qkv,
num_q_heads,
num_kv_heads,
num_kv_heads,
head_dim,
eps,
q_weight,
k_weight,
cos_sin_cache,
is_neox,
positions.view(-1),
)
return inputs
_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {}
def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None:
"""Pick the best pre-tuned config for the given input shape.
Selection strategy:
1. Find the closest q_heads among available configs
(exact match preferred).
2. Find the closest kv_heads among available configs
(exact match preferred).
3. Among the num_tokens values tuned for that q_heads and q_heads, pick
the smallest num_tokens >= the input's num_tokens. If the input is
larger than all available num_tokens, fall back to the largest.
"""
if not config_keys:
return None
qkv, q_heads, kv_heads, *_ = args
num_tokens = qkv.shape[0]
cache_key = (num_tokens, q_heads, kv_heads)
cached = _pick_cache.get(cache_key)
if cached is not None:
return cached
configs: dict[int, dict[int, list[int]]] = {}
for key in config_keys:
if key.is_default():
continue
configs.setdefault(key["q_heads"], {}).setdefault(key["kv_heads"], []).append(
key["num_tokens"]
)
if not configs:
return None
best_q_heads = min(configs, key=lambda s: abs(s - q_heads))
best_kv_heads = min(configs[best_q_heads], key=lambda s: abs(s - kv_heads))
available_num_tokens = sorted(configs[best_q_heads][best_kv_heads])
best_num_tokens = next(
(n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1]
)
result = CaseKey(
{
"q_heads": best_q_heads,
"kv_heads": best_kv_heads,
"num_tokens": best_num_tokens,
}
)
_pick_cache[cache_key] = result
return result
def fake_impl(
qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]
num_heads_q: int,
num_heads_k: int,
num_heads_v: int,
head_dim: int,
eps: float,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor, # [max_position, rotary_dim]
is_neox: bool,
position_ids: torch.Tensor, # [num_tokens],
forced_token_heads_per_warp: int = -1, # dummy
) -> None:
return
def baseline(
qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]
num_heads_q: int,
num_heads_k: int,
num_heads_v: int,
head_dim: int,
eps: float,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor, # [max_position, rotary_dim]
is_neox: bool,
position_ids: torch.Tensor, # [num_tokens],
forced_token_heads_per_warp: int = -1, # dummy
) -> None:
q_size = num_heads_q * head_dim
kv_size = num_heads_k * head_dim
q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
q_by_head = q.view(*q.shape[:-1], q.shape[-1] // head_dim, head_dim)
q_by_head = ir.ops.rms_norm(q_by_head, q_weight, eps)
q = q_by_head.view(q.shape)
k_by_head = k.view(*k.shape[:-1], k.shape[-1] // head_dim, head_dim)
k_by_head = ir.ops.rms_norm(k_by_head, k_weight, eps)
k = k_by_head.view(k.shape)
q, k = RotaryEmbedding.forward_static(
position_ids, q, k, head_dim, cos_sin_cache.shape[1], cos_sin_cache, is_neox
)
qkv[:, :q_size].copy_(q)
qkv[:, q_size : q_size + kv_size].copy_(k)
# Overwrite autotune_baseline_atol and autotune_baseline_rtol
# if too many configs failed due to baseline check during autotuning
@register_kernel(
mutates_args=["qkv"],
config_picker=pick_config,
input_generator=generate_inputs,
fake_impl=fake_impl,
helion_settings=helion.Settings(
autotune_baseline_fn=baseline,
autotune_baseline_atol=5e-2,
autotune_baseline_rtol=5e-2,
ignore_warnings=[helion.exc.TensorOperationInWrapper],
),
) # type: ignore[misc]
def fused_qk_norm_rope(
qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim]
num_heads_q: int,
num_heads_k: int,
num_heads_v: int,
head_dim: int,
eps: float,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor, # [max_position, rotary_dim]
is_neox: bool,
position_ids: torch.Tensor, # [num_tokens],
forced_token_heads_per_warp: int = -1, # dummy
) -> None:
assert qkv.ndim == 2
num_tokens = qkv.shape[0]
total_heads = num_heads_q + num_heads_k + num_heads_v
assert qkv.shape[1] == total_heads * head_dim
hl.specialize(qkv.shape[1])
assert cos_sin_cache.ndim == 2
max_position, rotary_dim = cos_sin_cache.shape
hl.specialize(max_position)
hl.specialize(rotary_dim)
assert rotary_dim % 2 == 0
assert rotary_dim <= head_dim
embed_dim = rotary_dim // 2
hl.specialize(num_heads_q)
hl.specialize(num_heads_k)
hl.specialize(num_heads_v)
hl.specialize(head_dim)
assert position_ids.ndim == 1 and position_ids.shape[0] == num_tokens
hl.specialize(position_ids.shape[0])
assert q_weight.ndim == 1 and q_weight.shape[0] == head_dim
hl.specialize(q_weight.shape[0])
assert k_weight.ndim == 1 and k_weight.shape[0] == head_dim
hl.specialize(k_weight.shape[0])
assert qkv.dtype == q_weight.dtype and q_weight.dtype == k_weight.dtype
assert position_ids.dtype == torch.int64
assert qkv.is_contiguous()
assert position_ids.is_contiguous()
assert q_weight.is_contiguous()
assert k_weight.is_contiguous()
assert cos_sin_cache.is_contiguous()
qk_heads = num_heads_q + num_heads_k
qkv = qkv.view(num_tokens, -1, head_dim)
for tile_m, tile_gn, tile_n in hl.tile(
[num_tokens, qk_heads, head_dim], block_size=[1, None, head_dim]
):
x_blk = qkv[tile_m, tile_gn, tile_n].to(dtype=torch.float32)
rms = x_blk.pow(2).sum(dim=-1)
rms = torch.rsqrt(rms * (1.0 / head_dim) + eps)
use_q_weight = (tile_gn.index < num_heads_q)[None, :, None]
w_blk = torch.where(
use_q_weight, q_weight[None, None, tile_n], k_weight[None, None, tile_n]
)
x_blk = (x_blk * rms[:, :, None]).to(qkv.dtype) * w_blk
qkv[tile_m, tile_gn, tile_n] = x_blk
pos_id = position_ids[tile_m]
cos_blk = cos_sin_cache[pos_id, hl.arange(embed_dim)]
sin_blk = cos_sin_cache[pos_id, hl.arange(embed_dim) + embed_dim]
if is_neox:
x1_offset = hl.arange(embed_dim)
x2_offset = x1_offset + embed_dim
else:
x1_offset = hl.arange(embed_dim) * 2
x2_offset = x1_offset + 1
x1_blk = qkv[tile_m, tile_gn, x1_offset]
x2_blk = qkv[tile_m, tile_gn, x2_offset]
o1_blk = x1_blk * cos_blk[:, None, :] - x2_blk * sin_blk[:, None, :]
o2_blk = x2_blk * cos_blk[:, None, :] + x1_blk * sin_blk[:, None, :]
qkv[tile_m, tile_gn, x1_offset] = o1_blk
qkv[tile_m, tile_gn, x2_offset] = o2_blk