diff --git a/tests/kernels/helion/test_fused_qk_norm_rope.py b/tests/kernels/helion/test_fused_qk_norm_rope.py new file mode 100644 index 0000000000..19d2fc9b5a --- /dev/null +++ b/tests/kernels/helion/test_fused_qk_norm_rope.py @@ -0,0 +1,261 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the fused_qk_norm_rope helion kernel + +Run `pytest tests/kernels/helion/test_fused_qk_norm_rope.py`. +""" + +from typing import Any + +import pytest +import torch +from torch._subclasses.fake_tensor import FakeTensorMode + +from tests.kernels.helion.utils import skip_if_platform_unsupported +from vllm.benchmarks.lib.utils import default_vllm_config +from vllm.kernels.helion.case_key import CaseKey +from vllm.kernels.helion.config_manager import ConfigManager +from vllm.kernels.helion.ops.fused_qk_norm_rope import ( + _pick_cache, + baseline, + fused_qk_norm_rope, + pick_config, +) +from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding +from vllm.utils.import_utils import has_helion + +if not has_helion(): + pytest.skip( + "Helion is not installed. Install with: pip install vllm[helion]", + allow_module_level=True, + ) + + +@default_vllm_config() +def _generate_fake_input( + num_tokens: int, num_q_heads: int, num_kv_heads: int +) -> tuple[Any, ...]: + with FakeTensorMode(): + head_dim = 128 + eps = 1e-6 + is_neox = True + rotary_ratio = 1.0 + device = "cuda" + dtype = torch.bfloat16 + total_dim = (num_q_heads + 2 * num_kv_heads) * head_dim + qkv = torch.randn(num_tokens, total_dim, dtype=dtype, device=device) + positions = torch.arange(num_tokens, dtype=torch.long, device=device) + q_weight = torch.normal( + mean=1.0, + std=1.0, + size=(head_dim,), + dtype=qkv.dtype, + device=device, + ) + k_weight = torch.normal( + mean=1.0, + std=1.0, + size=(head_dim,), + dtype=qkv.dtype, + device=device, + ) + rotary_dim = int(head_dim * rotary_ratio) + rope = RotaryEmbedding( + head_size=head_dim, + rotary_dim=rotary_dim, + max_position_embeddings=4096, + base=10000.0, + is_neox_style=is_neox, + dtype=dtype, + ).to(device) + args = ( + qkv, + num_q_heads, + num_kv_heads, + num_kv_heads, + head_dim, + eps, + q_weight, + k_weight, + rope.cos_sin_cache, + is_neox, + positions.view(-1), + ) + return args + + +@pytest.fixture(autouse=True) +def reset_config_manager_singleton(): + ConfigManager.reset_instance() + ConfigManager() + yield + ConfigManager.reset_instance() + + +class TestFusedQkNormRopeConfigPicker: + def setup_method(self): + _pick_cache.clear() + + def test_config_picker_exact_match(self): + config_keys = [ + CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}), + CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}), + ] + + args = _generate_fake_input(16, 4096, 128) + selected_key = pick_config(args, config_keys) + assert selected_key == CaseKey( + {"q_heads": 4096, "kv_heads": 128, "num_tokens": 16} + ) + + def test_config_picker_closest_match(self): + config_keys = [ + CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}), + CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}), + CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 16}), + CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 32}), + CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 16}), + CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 32}), + CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}), + CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}), + ] + + args = _generate_fake_input(20, 3000, 70) + selected_key = pick_config(args, config_keys) + assert selected_key == CaseKey( + {"q_heads": 2048, "kv_heads": 64, "num_tokens": 32} + ) + + def test_config_picker_no_configs(self): + config_keys: list[dict] = [] + + args = _generate_fake_input(16, 4096, 128) + selected_key = pick_config(args, config_keys) + assert selected_key is None + + def test_config_picker_fallback_to_largest(self): + config_keys = [ + CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}), + CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}), + CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 16}), + CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 32}), + CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 16}), + CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 32}), + CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}), + CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}), + ] + + args = _generate_fake_input(64, 8192, 256) + selected_key = pick_config(args, config_keys) + assert selected_key == CaseKey( + {"q_heads": 4096, "kv_heads": 128, "num_tokens": 32} + ) + + +class TestFusedQkNormRopeCorrectness: + @pytest.mark.parametrize( + "num_heads, num_kv_heads, head_dim", [(16, 4, 128), (64, 8, 128)] + ) + @pytest.mark.parametrize("num_tokens", [1, 7, 1024, 1025]) + @pytest.mark.parametrize("is_neox", [False, True]) + @pytest.mark.parametrize("rotary_ratio", [1.0, 0.5, 0.25]) + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) + @default_vllm_config() + def test_fused_qk_norm_rope( + self, + num_heads: int, + num_kv_heads: int, + head_dim: int, + num_tokens: int, + is_neox: bool, + rotary_ratio: float, + dtype: torch.dtype, + ): + skip_if_platform_unsupported("fused_qk_norm_rope") + + torch.manual_seed(42) + eps = 1e-6 + device = "cuda" + total_dim = (num_heads + 2 * num_kv_heads) * head_dim + ref_qkv = torch.empty( + num_tokens, total_dim, dtype=dtype, device=device + ).uniform_(-0.1, 0.1) + ops_qkv = ref_qkv.clone() + positions = torch.arange(num_tokens, dtype=torch.long, device=device) + q_weight = torch.empty(head_dim, dtype=dtype, device=device).uniform_(0.8, 1.2) + k_weight = torch.empty(head_dim, dtype=dtype, device=device).uniform_(0.8, 1.2) + rotary_dim = int(head_dim * rotary_ratio) + rope = RotaryEmbedding( + head_size=head_dim, + rotary_dim=rotary_dim, + max_position_embeddings=40960, + base=10000.0, + is_neox_style=is_neox, + dtype=dtype, + ).to(device) + + baseline( + ref_qkv, + num_heads, + num_kv_heads, + num_kv_heads, + head_dim, + eps, + q_weight, + k_weight, + rope.cos_sin_cache, + is_neox, + positions.view(-1), + ) + + fused_qk_norm_rope( + ops_qkv, + num_heads, + num_kv_heads, + num_kv_heads, + head_dim, + eps, + q_weight, + k_weight, + rope.cos_sin_cache, + is_neox, + positions.view(-1), + ) + + if dtype == torch.bfloat16: + atol = 5e-2 + rtol = 5e-2 + else: + atol = 1e-2 + rtol = 1e-2 + + torch.testing.assert_close( + ref_qkv, + ops_qkv, + atol=atol, + rtol=rtol, + ) + + +class TestFusedQkNormRopeIntegration: + def test_kernel_registration_integration(self): + from vllm.kernels.helion.register import get_registered_kernels + + registered_kernels = get_registered_kernels() + assert "fused_qk_norm_rope" in registered_kernels + + kernel_wrapper = registered_kernels["fused_qk_norm_rope"] + assert kernel_wrapper.op_name == "fused_qk_norm_rope" + assert kernel_wrapper._config_picker is not None + assert kernel_wrapper._mutates_args == ["qkv"] + + def test_fake_impl_functionality(self): + skip_if_platform_unsupported("fused_qk_norm_rope") + from vllm.kernels.helion.register import get_registered_kernels + + registered_kernels = get_registered_kernels() + kernel_wrapper = registered_kernels["fused_qk_norm_rope"] + fake_impl = kernel_wrapper._fake_impl + + args = _generate_fake_input(16, 4096, 128) + assert fake_impl(*args) is None diff --git a/vllm/kernels/helion/configs/fused_qk_norm_rope/nvidia_b200.json b/vllm/kernels/helion/configs/fused_qk_norm_rope/nvidia_b200.json new file mode 100644 index 0000000000..cf806f0a4d --- /dev/null +++ b/vllm/kernels/helion/configs/fused_qk_norm_rope/nvidia_b200.json @@ -0,0 +1,2612 @@ +[ + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 8 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "", + "last", + "first", + "", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 0, + 1, + 2 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "last", + "first", + "first", + "" + ], + "num_warps": 4, + "num_stages": 3, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "first", + "", + "first", + "last", + "first", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 1 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + true + ], + "range_flattens": [ + true + ], + "load_eviction_policies": [ + "last", + "first", + "", + "last", + "first", + "last", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_interleaved", + "num_sm_multiplier": 4, + "maxnreg": 128 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "first", + "", + "first", + "last", + "first", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "first", + "", + "first", + "last", + "first", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 8 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + true + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "last", + "first", + "last", + "last", + "first", + "last", + "", + "last" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 4, + "maxnreg": 128 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 0, + 1, + 2 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "last", + "first", + "first", + "first", + "", + "first" + ], + "num_warps": 1, + "num_stages": 3, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "first", + "", + "first", + "last", + "first", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 0, + 2, + 1 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "", + "", + "last", + "last", + "first", + "first" + ], + "num_warps": 4, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "first", + "", + "first", + "last", + "first", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 8 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "first", + "first", + "first", + "last", + "last", + "last", + "last" + ], + "num_warps": 8, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "first", + "", + "first", + "", + "", + "first" + ], + "num_warps": 2, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "last", + "first", + "", + "first", + "first", + "", + "first" + ], + "num_warps": 2, + "num_stages": 4, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 0, + 2, + 1 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "", + "", + "last", + "last", + "first", + "first" + ], + "num_warps": 4, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "first", + "", + "first", + "", + "", + "first" + ], + "num_warps": 2, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "last", + "last", + "", + "last", + "last", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "first", + "", + "first", + "", + "", + "first" + ], + "num_warps": 2, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "first", + "", + "first", + "", + "", + "first" + ], + "num_warps": 2, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "first", + "", + "first", + "last", + "first", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 0, + 2, + 1 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "last", + "first", + "", + "last", + "first", + "first", + "first" + ], + "num_warps": 2, + "num_stages": 7, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "first", + "", + "first", + "", + "", + "first" + ], + "num_warps": 2, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 8 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "", + "", + "last", + "last", + "first", + "" + ], + "num_warps": 1, + "num_stages": 7, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 8 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "", + "last", + "first", + "", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "first", + "", + "first", + "", + "", + "first" + ], + "num_warps": 2, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "last", + "first", + "", + "first", + "first", + "", + "first" + ], + "num_warps": 2, + "num_stages": 4, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "last", + "last", + "", + "last", + "last", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 8 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last", + "", + "last", + "last", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 2 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "first", + "first", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 32, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 4 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "last", + "", + "first", + "first", + "", + "first", + "last", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 32 + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 2 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "first", + "first", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 32, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 2 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + true + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "last", + "", + "", + "last", + "last", + "first", + "" + ], + "num_warps": 1, + "num_stages": 7, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_interleaved", + "num_sm_multiplier": 32 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 2 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + true + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "last", + "", + "", + "last", + "last", + "first", + "" + ], + "num_warps": 1, + "num_stages": 7, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_interleaved", + "num_sm_multiplier": 32 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 2 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + true + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "first", + "last", + "", + "first", + "", + "first", + "" + ], + "num_warps": 1, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_interleaved", + "num_sm_multiplier": 32 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + null + ], + "range_multi_buffers": [ + true + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "", + "last", + "last" + ], + "num_warps": 1, + "num_stages": 6, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [ + false + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + } +] \ No newline at end of file diff --git a/vllm/kernels/helion/configs/fused_qk_norm_rope/nvidia_h100.json b/vllm/kernels/helion/configs/fused_qk_norm_rope/nvidia_h100.json new file mode 100644 index 0000000000..825c63ca35 --- /dev/null +++ b/vllm/kernels/helion/configs/fused_qk_norm_rope/nvidia_h100.json @@ -0,0 +1,2722 @@ +[ + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 1 + ], + "loop_orders": [ + [ + 1, + 0, + 2 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 4 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + false + ], + "range_flattens": [ + true + ], + "load_eviction_policies": [ + "", + "", + "first", + "", + "", + "last", + "", + "first" + ], + "num_warps": 1, + "num_stages": 3, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 2 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 1 + ], + "loop_orders": [ + [ + 0, + 2, + 1 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first", + "first", + "last", + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 0, + 1 + ] + ], + "l2_groupings": [ + 8 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "first", + "first", + "last", + "last", + "first" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 1 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 16 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "", + "first", + "first", + "last", + "first", + "last" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 1 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last", + "first", + "first", + "first", + "last", + "" + ], + "num_warps": 1, + "num_stages": 8, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "first", + "", + "", + "last", + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 1 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 16 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "last", + "", + "", + "last", + "last", + "", + "last" + ], + "num_warps": 1, + "num_stages": 4, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "last", + "", + "first", + "", + "first", + "last" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "first", + "last", + "last", + "last", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 1 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 16 + ], + "range_unroll_factors": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "first", + "first", + "last", + "first", + "first", + "last", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat", + "atomic_indexing": [], + "range_warp_specializes": [], + "range_num_stages": [] + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "first", + "last", + "last", + "last", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "first", + "last", + "last", + "last", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 1 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 16 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "last", + "", + "", + "last", + "last", + "", + "last" + ], + "num_warps": 1, + "num_stages": 4, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "last", + "first", + "last", + "last", + "last", + "" + ], + "num_warps": 1, + "num_stages": 8, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "last", + "", + "first", + "last", + "last", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 4, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "", + "first", + "last", + "", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 32 + ], + "range_unroll_factors": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "", + "first", + "last", + "", + "last", + "last" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat", + "atomic_indexing": [], + "range_warp_specializes": [], + "range_num_stages": [] + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 8 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "last", + "last", + "first", + "last", + "first", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 8 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last", + "", + "last", + "last", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "first", + "last", + "", + "last", + "last", + "last", + "first" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "last", + "last", + "last", + "last", + "first", + "first", + "" + ], + "num_warps": 2, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 8 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last", + "", + "last", + "last", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "last", + "" + ], + "num_warps": 2, + "num_stages": 5, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 8 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "", + "", + "first", + "", + "first", + "first" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 4 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "last", + "" + ], + "num_warps": 2, + "num_stages": 5, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 2 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "first", + "first", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 32, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 2 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "first", + "first", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 32, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 2 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "first", + "first", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 32, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 2 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "first", + "first", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 32, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 4 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + true + ], + "load_eviction_policies": [ + "first", + "", + "first", + "first", + "last", + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_interleaved", + "num_sm_multiplier": 64, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 2 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "first", + "first", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 32, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 64 + ], + "range_unroll_factors": [ + 2 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "first", + "first", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 5, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 32, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "first", + "last", + "", + "last", + "", + "first" + ], + "num_warps": 1, + "num_stages": 6, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "persistent_interleaved", + "num_sm_multiplier": 128, + "maxnreg": 64, + "atomic_indexing": [], + "range_warp_specializes": [] + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 16 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + false + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "", + "", + "last", + "", + "last", + "first", + "" + ], + "num_warps": 1, + "num_stages": 4, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 64, + "maxnreg": 256 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 16 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + true + ], + "load_eviction_policies": [ + "", + "first", + "first", + "last", + "", + "", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 64, + "maxnreg": 128 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 16 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + true + ], + "load_eviction_policies": [ + "", + "first", + "first", + "last", + "", + "", + "last", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 64, + "maxnreg": 128 + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 16 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + false + ], + "range_flattens": [ + false + ], + "load_eviction_policies": [ + "", + "", + "", + "last", + "", + "last", + "first", + "" + ], + "num_warps": 1, + "num_stages": 4, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 64, + "maxnreg": 256 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + false + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last", + "", + "", + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + false + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last", + "", + "", + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + false + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last", + "", + "", + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + false + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last", + "", + "", + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + false + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last", + "", + "", + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 16, + "kv_heads": 8, + "num_tokens": 16384 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 1, + 2, + 0 + ] + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + true + ], + "load_eviction_policies": [ + "", + "last", + "last", + "", + "last", + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 7, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64, + "atomic_indexing": [], + "range_warp_specializes": [] + } + }, + { + "key": { + "q_heads": 32, + "kv_heads": 8, + "num_tokens": 16384 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + false + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last", + "", + "", + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + }, + { + "key": { + "q_heads": 64, + "kv_heads": 8, + "num_tokens": 16384 + }, + "config": { + "block_sizes": [ + 2 + ], + "loop_orders": [ + [ + 2, + 1, + 0 + ] + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + false + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last", + "", + "", + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_blocked", + "num_sm_multiplier": 128, + "maxnreg": 64 + } + } +] \ No newline at end of file diff --git a/vllm/kernels/helion/ops/fused_qk_norm_rope.py b/vllm/kernels/helion/ops/fused_qk_norm_rope.py new file mode 100644 index 0000000000..c97ad1e514 --- /dev/null +++ b/vllm/kernels/helion/ops/fused_qk_norm_rope.py @@ -0,0 +1,316 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from itertools import product +from typing import Any + +import torch + +from vllm.kernels.helion.case_key import CaseKey +from vllm.logger import init_logger +from vllm.utils.import_utils import has_helion + +if not has_helion(): + raise ImportError( + "Helion kernel requires helion to be installed. " + "Install it with: pip install helion" + ) + +import helion +import helion.language as hl + +from vllm import ir +from vllm.kernels.helion.register import register_kernel +from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding + +logger = init_logger(__name__) + + +def _compute_cos_sin_cache( + max_position_embeddings, rotary_dim, device="cuda", dtype=torch.float +): + inv_freq = 1.0 / ( + 10000 + ** (torch.arange(0, rotary_dim, 2, device=device, dtype=dtype) / rotary_dim) + ) + + t = torch.arange(max_position_embeddings, device=device, dtype=dtype) + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + return cache + + +def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]: + # TODO(xiaohongchen1991): it is difficult for kernel author to cover + # all input property combination. Currently, dtypes are fixed. We need + # optimization to bucket/skip some combinations + num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192] + num_heads_pair = [ + (16, 8), + (32, 8), + (64, 8), + ] + head_dim = 128 + in_dtype: torch.dtype = torch.bfloat16 + rotary_ratio = 1.0 + is_neox = True + eps = 1e-6 + device = "cuda" + inputs = {} + + for num_tokens, (num_q_heads, num_kv_heads) in product( + num_tokens_list, num_heads_pair + ): + total_dim = (num_q_heads + 2 * num_kv_heads) * head_dim + qkv = torch.empty( + num_tokens, total_dim, dtype=in_dtype, device=device + ).uniform_(-0.1, 0.1) + positions = torch.arange(num_tokens, dtype=torch.long, device=device) + q_weight = torch.empty(head_dim, dtype=in_dtype, device=device).uniform_( + 0.8, 1.2 + ) + k_weight = torch.empty(head_dim, dtype=in_dtype, device=device).uniform_( + 0.8, 1.2 + ) + rotary_dim = int(head_dim * rotary_ratio) + cos_sin_cache = _compute_cos_sin_cache(40960, rotary_dim) + cos_sin_cache = cos_sin_cache.to(in_dtype) + + config_key = CaseKey( + { + "q_heads": num_q_heads, + "kv_heads": num_kv_heads, + "num_tokens": num_tokens, + } + ) + inputs[config_key] = ( + qkv, + num_q_heads, + num_kv_heads, + num_kv_heads, + head_dim, + eps, + q_weight, + k_weight, + cos_sin_cache, + is_neox, + positions.view(-1), + ) + + return inputs + + +_pick_cache: dict[tuple[int, int, int], CaseKey | None] = {} + + +def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None: + """Pick the best pre-tuned config for the given input shape. + + Selection strategy: + 1. Find the closest q_heads among available configs + (exact match preferred). + 2. Find the closest kv_heads among available configs + (exact match preferred). + 3. Among the num_tokens values tuned for that q_heads and q_heads, pick + the smallest num_tokens >= the input's num_tokens. If the input is + larger than all available num_tokens, fall back to the largest. + """ + + if not config_keys: + return None + + qkv, q_heads, kv_heads, *_ = args + num_tokens = qkv.shape[0] + + cache_key = (num_tokens, q_heads, kv_heads) + cached = _pick_cache.get(cache_key) + if cached is not None: + return cached + + configs: dict[int, dict[int, list[int]]] = {} + for key in config_keys: + if key.is_default(): + continue + configs.setdefault(key["q_heads"], {}).setdefault(key["kv_heads"], []).append( + key["num_tokens"] + ) + + if not configs: + return None + + best_q_heads = min(configs, key=lambda s: abs(s - q_heads)) + best_kv_heads = min(configs[best_q_heads], key=lambda s: abs(s - kv_heads)) + available_num_tokens = sorted(configs[best_q_heads][best_kv_heads]) + best_num_tokens = next( + (n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1] + ) + + result = CaseKey( + { + "q_heads": best_q_heads, + "kv_heads": best_kv_heads, + "num_tokens": best_num_tokens, + } + ) + _pick_cache[cache_key] = result + return result + + +def fake_impl( + qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim] + num_heads_q: int, + num_heads_k: int, + num_heads_v: int, + head_dim: int, + eps: float, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, # [max_position, rotary_dim] + is_neox: bool, + position_ids: torch.Tensor, # [num_tokens], + forced_token_heads_per_warp: int = -1, # dummy +) -> None: + return + + +def baseline( + qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim] + num_heads_q: int, + num_heads_k: int, + num_heads_v: int, + head_dim: int, + eps: float, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, # [max_position, rotary_dim] + is_neox: bool, + position_ids: torch.Tensor, # [num_tokens], + forced_token_heads_per_warp: int = -1, # dummy +) -> None: + q_size = num_heads_q * head_dim + kv_size = num_heads_k * head_dim + + q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1) + + q_by_head = q.view(*q.shape[:-1], q.shape[-1] // head_dim, head_dim) + q_by_head = ir.ops.rms_norm(q_by_head, q_weight, eps) + q = q_by_head.view(q.shape) + + k_by_head = k.view(*k.shape[:-1], k.shape[-1] // head_dim, head_dim) + k_by_head = ir.ops.rms_norm(k_by_head, k_weight, eps) + k = k_by_head.view(k.shape) + + q, k = RotaryEmbedding.forward_static( + position_ids, q, k, head_dim, cos_sin_cache.shape[1], cos_sin_cache, is_neox + ) + qkv[:, :q_size].copy_(q) + qkv[:, q_size : q_size + kv_size].copy_(k) + + +# Overwrite autotune_baseline_atol and autotune_baseline_rtol +# if too many configs failed due to baseline check during autotuning +@register_kernel( + mutates_args=["qkv"], + config_picker=pick_config, + input_generator=generate_inputs, + fake_impl=fake_impl, + helion_settings=helion.Settings( + autotune_baseline_fn=baseline, + autotune_baseline_atol=5e-2, + autotune_baseline_rtol=5e-2, + ignore_warnings=[helion.exc.TensorOperationInWrapper], + ), +) # type: ignore[misc] +def fused_qk_norm_rope( + qkv: torch.Tensor, # [num_tokens, (num_heads_q+num_heads_k+num_heads_v)*head_dim] + num_heads_q: int, + num_heads_k: int, + num_heads_v: int, + head_dim: int, + eps: float, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, # [max_position, rotary_dim] + is_neox: bool, + position_ids: torch.Tensor, # [num_tokens], + forced_token_heads_per_warp: int = -1, # dummy +) -> None: + assert qkv.ndim == 2 + num_tokens = qkv.shape[0] + total_heads = num_heads_q + num_heads_k + num_heads_v + assert qkv.shape[1] == total_heads * head_dim + hl.specialize(qkv.shape[1]) + + assert cos_sin_cache.ndim == 2 + max_position, rotary_dim = cos_sin_cache.shape + hl.specialize(max_position) + hl.specialize(rotary_dim) + assert rotary_dim % 2 == 0 + assert rotary_dim <= head_dim + embed_dim = rotary_dim // 2 + + hl.specialize(num_heads_q) + hl.specialize(num_heads_k) + hl.specialize(num_heads_v) + hl.specialize(head_dim) + + assert position_ids.ndim == 1 and position_ids.shape[0] == num_tokens + hl.specialize(position_ids.shape[0]) + + assert q_weight.ndim == 1 and q_weight.shape[0] == head_dim + hl.specialize(q_weight.shape[0]) + assert k_weight.ndim == 1 and k_weight.shape[0] == head_dim + hl.specialize(k_weight.shape[0]) + + assert qkv.dtype == q_weight.dtype and q_weight.dtype == k_weight.dtype + assert position_ids.dtype == torch.int64 + + assert qkv.is_contiguous() + assert position_ids.is_contiguous() + assert q_weight.is_contiguous() + assert k_weight.is_contiguous() + assert cos_sin_cache.is_contiguous() + + qk_heads = num_heads_q + num_heads_k + + qkv = qkv.view(num_tokens, -1, head_dim) + + for tile_m, tile_gn, tile_n in hl.tile( + [num_tokens, qk_heads, head_dim], block_size=[1, None, head_dim] + ): + x_blk = qkv[tile_m, tile_gn, tile_n].to(dtype=torch.float32) + + rms = x_blk.pow(2).sum(dim=-1) + rms = torch.rsqrt(rms * (1.0 / head_dim) + eps) + + use_q_weight = (tile_gn.index < num_heads_q)[None, :, None] + w_blk = torch.where( + use_q_weight, q_weight[None, None, tile_n], k_weight[None, None, tile_n] + ) + + x_blk = (x_blk * rms[:, :, None]).to(qkv.dtype) * w_blk + + qkv[tile_m, tile_gn, tile_n] = x_blk + + pos_id = position_ids[tile_m] + cos_blk = cos_sin_cache[pos_id, hl.arange(embed_dim)] + sin_blk = cos_sin_cache[pos_id, hl.arange(embed_dim) + embed_dim] + + if is_neox: + x1_offset = hl.arange(embed_dim) + x2_offset = x1_offset + embed_dim + else: + x1_offset = hl.arange(embed_dim) * 2 + x2_offset = x1_offset + 1 + + x1_blk = qkv[tile_m, tile_gn, x1_offset] + x2_blk = qkv[tile_m, tile_gn, x2_offset] + + o1_blk = x1_blk * cos_blk[:, None, :] - x2_blk * sin_blk[:, None, :] + o2_blk = x2_blk * cos_blk[:, None, :] + x1_blk * sin_blk[:, None, :] + + qkv[tile_m, tile_gn, x1_offset] = o1_blk + qkv[tile_m, tile_gn, x2_offset] = o2_blk