diff --git a/tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py b/tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py new file mode 100644 index 0000000000..3842419562 --- /dev/null +++ b/tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py @@ -0,0 +1,207 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the rms_norm_dynamic_per_token_quant helion kernel + +Run `pytest tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py`. +""" + +from typing import Any + +import pytest +import torch +from torch._subclasses.fake_tensor import FakeTensorMode + +from tests.kernels.helion.utils import skip_if_platform_unsupported +from vllm.kernels.helion.case_key import CaseKey +from vllm.kernels.helion.config_manager import ConfigManager +from vllm.kernels.helion.ops.rms_norm_dynamic_per_token_quant import ( + _pick_cache, + baseline, + pick_config, + rms_norm_dynamic_per_token_quant, +) +from vllm.platforms import current_platform +from vllm.utils.import_utils import has_helion +from vllm.utils.torch_utils import set_random_seed + +if not has_helion(): + pytest.skip( + "Helion is not installed. Install with: pip install vllm[helion]", + allow_module_level=True, + ) + + +def _generate_fake_input(num_tokens: int, hidden_size: int) -> tuple[Any, ...]: + with FakeTensorMode(): + input = torch.randn( + num_tokens, hidden_size, device="cuda", dtype=torch.bfloat16 + ) + result = torch.empty( + input.shape, device=input.device, dtype=current_platform.fp8_dtype() + ) + scale = torch.empty((num_tokens, 1), device=input.device, dtype=torch.float32) + scale_ub = torch.mean(input).to(torch.float32) + residual = torch.randn_like(input) + weight = torch.normal( + mean=1.0, + std=1.0, + size=(hidden_size,), + dtype=input.dtype, + device=input.device, + ) + epsilon = 1e-6 + args = (result, input, weight, scale, epsilon, scale_ub, residual) + return args + + +@pytest.fixture(autouse=True) +def reset_config_manager_singleton(): + ConfigManager.reset_instance() + ConfigManager() + yield + ConfigManager.reset_instance() + + +class TestRmsNormDynamicPerTokenQuantConfigPicker: + def setup_method(self): + _pick_cache.clear() + + def test_config_picker_exact_match(self): + config_keys = [ + CaseKey({"hidden_size": 2048, "num_tokens": 16}), + CaseKey({"hidden_size": 4096, "num_tokens": 16}), + ] + + args = _generate_fake_input(16, 4096) + selected_key = pick_config(args, config_keys) + assert selected_key == CaseKey({"hidden_size": 4096, "num_tokens": 16}) + + def test_config_picker_closest_match(self): + config_keys = [ + CaseKey({"hidden_size": 2048, "num_tokens": 16}), + CaseKey({"hidden_size": 2048, "num_tokens": 32}), + CaseKey({"hidden_size": 4096, "num_tokens": 16}), + CaseKey({"hidden_size": 4096, "num_tokens": 32}), + ] + + args = _generate_fake_input(20, 3000) + selected_key = pick_config(args, config_keys) + assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32}) + + def test_config_picker_no_configs(self): + config_keys: list[dict] = [] + + args = _generate_fake_input(16, 4096) + selected_key = pick_config(args, config_keys) + assert selected_key is None + + def test_config_picker_fallback_to_largest(self): + config_keys = [ + CaseKey({"hidden_size": 2048, "num_tokens": 16}), + CaseKey({"hidden_size": 4096, "num_tokens": 16}), + ] + + args = _generate_fake_input(32, 8192) + selected_key = pick_config(args, config_keys) + assert selected_key == CaseKey({"hidden_size": 4096, "num_tokens": 16}) + + +DTYPES = [torch.bfloat16, torch.float] +QUANT_DTYPES = [torch.int8, current_platform.fp8_dtype()] +VEC_HIDDEN_SIZES = [1024, 1025, 1027, 1029] +# Avoid combinatorial explosion with full Cartesian product +NUM_TOKENS_HIDDEN_SIZES = [ + *[(1, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5120, 5137]], + *[(2048, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5137]], + *[(4096, i) for i in [1, 64, 5137]], +] + +ADD_RESIDUAL = [False, True] +SCALE_UBS = [True, False] +SEEDS = [0] + +EPS = 1e-6 + + +class TestRmsNormDynamicPerTokenQuantCorrectness: + @pytest.mark.parametrize("num_tokens, hidden_size", NUM_TOKENS_HIDDEN_SIZES) + @pytest.mark.parametrize("add_residual", ADD_RESIDUAL) + @pytest.mark.parametrize("has_scale_ub", SCALE_UBS) + @pytest.mark.parametrize("dtype", DTYPES) + @pytest.mark.parametrize("quant_dtype", QUANT_DTYPES) + @pytest.mark.parametrize("seed", SEEDS) + def test_rms_norm_dynamic_per_token_quant( + self, + num_tokens: int, + hidden_size: int, + add_residual: bool, + has_scale_ub: bool, + dtype: torch.dtype, + quant_dtype: torch.dtype, + seed: int, + ) -> None: + skip_if_platform_unsupported("rms_norm_dynamic_per_token_quant") + + set_random_seed(seed) + + if has_scale_ub and quant_dtype != current_platform.fp8_dtype(): + # skip + return + + scale = 1 / (hidden_size) + x = torch.randn(num_tokens, hidden_size, dtype=dtype, device="cuda") * scale + weight = torch.normal( + mean=1.0, std=1.0, size=(hidden_size,), dtype=dtype, device=x.device + ) + residual = torch.randn_like(x) * scale if add_residual else None + scale_ub = ( + torch.mean(x).to(dtype=torch.float32, device="cuda") + if has_scale_ub + else None + ) + + ref_out = torch.empty(x.shape, device=x.device, dtype=quant_dtype) + ref_scales = torch.empty((x.shape[0], 1), device=x.device, dtype=torch.float32) + ref_residual = residual.clone() if residual is not None else None + baseline(ref_out, x, weight, ref_scales, EPS, scale_ub, ref_residual) + + ops_out = torch.empty(x.shape, device=x.device, dtype=quant_dtype) + ops_scales = torch.empty((x.shape[0], 1), device=x.device, dtype=torch.float32) + ops_residual = residual.clone() if residual is not None else None + rms_norm_dynamic_per_token_quant( + ops_out, x, weight, ops_scales, EPS, scale_ub, ops_residual + ) + + torch.testing.assert_close(ref_scales, ops_scales) + # allow 1 ULP difference + assert ( + ref_out.view(torch.uint8).to(torch.int16) + - ops_out.view(torch.uint8).to(torch.int16) + ).abs().max() <= 1 + + if add_residual: + torch.testing.assert_close(ref_residual, ops_residual) + + +class TestRmsNormDynamicPerTokenQuantIntegration: + def test_kernel_registration_integration(self): + from vllm.kernels.helion.register import get_registered_kernels + + registered_kernels = get_registered_kernels() + assert "rms_norm_dynamic_per_token_quant" in registered_kernels + + kernel_wrapper = registered_kernels["rms_norm_dynamic_per_token_quant"] + assert kernel_wrapper.op_name == "rms_norm_dynamic_per_token_quant" + assert kernel_wrapper._config_picker is not None + assert kernel_wrapper._mutates_args == ["result", "scale", "residual"] + + def test_fake_impl_functionality(self): + skip_if_platform_unsupported("rms_norm_dynamic_per_token_quant") + from vllm.kernels.helion.register import get_registered_kernels + + registered_kernels = get_registered_kernels() + kernel_wrapper = registered_kernels["rms_norm_dynamic_per_token_quant"] + fake_impl = kernel_wrapper._fake_impl + + args = _generate_fake_input(16, 4096) + assert fake_impl(*args) is None diff --git a/vllm/kernels/helion/configs/rms_norm_dynamic_per_token_quant/nvidia_b200.json b/vllm/kernels/helion/configs/rms_norm_dynamic_per_token_quant/nvidia_b200.json new file mode 100644 index 0000000000..5be0d7b71b --- /dev/null +++ b/vllm/kernels/helion/configs/rms_norm_dynamic_per_token_quant/nvidia_b200.json @@ -0,0 +1,2647 @@ +[ + { + "key": { + "hidden_size": 2048, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 2, + 4, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + false + ], + "range_flattens": [ + null, + false, + false, + true + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "last", + "", + "first", + "last", + "", + "" + ], + "num_warps": 16, + "num_stages": 3, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 2, + 3, + 1, + 0 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + true, + null, + false, + null + ], + "range_flattens": [ + null, + null, + false, + true + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 16, + "num_stages": 7, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_interleaved", + "num_sm_multiplier": 1, + "maxnreg": 128 + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 2, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + false + ], + "range_flattens": [ + null, + false, + true, + false + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "", + "last", + "last", + "first", + "first" + ], + "num_warps": 32, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 0, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + null + ], + "range_flattens": [ + null, + true, + null, + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 7, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + false, + null + ], + "range_flattens": [ + null, + false, + true, + true + ], + "load_eviction_policies": [ + "", + "first", + "", + "first", + "last", + "", + "last", + "", + "last" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 1, + 2, + 0 + ], + "range_warp_specializes": [ + null, + null, + null, + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + false, + true + ], + "range_flattens": [ + null, + null, + true, + null + ], + "load_eviction_policies": [ + "", + "first", + "first", + "", + "last", + "", + "", + "last", + "" + ], + "num_warps": 8, + "num_stages": 3, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 0, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + null + ], + "range_flattens": [ + null, + true, + null, + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 7, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 3, + 1 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + true, + null + ], + "range_flattens": [ + null, + false, + false, + true + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "last", + "first", + "first", + "", + "" + ], + "num_warps": 8, + "num_stages": 4, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 1, + 2, + 1 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + null, + null + ], + "range_flattens": [ + null, + null, + null, + null + ], + "load_eviction_policies": [ + "first", + "", + "first", + "", + "last", + "first", + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 7, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + true, + true + ], + "range_flattens": [ + null, + false, + false, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 2, + 0, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + true, + true, + false + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "last", + "last", + "last", + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 4, + 1 + ], + "range_warp_specializes": [ + null, + false, + null, + false + ], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + false + ], + "range_flattens": [ + null, + true, + false, + true + ], + "load_eviction_policies": [ + "last", + "", + "last", + "first", + "first", + "", + "last", + "", + "last" + ], + "num_warps": 8, + "num_stages": 7, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + true, + true + ], + "range_flattens": [ + null, + false, + false, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 1, + 2, + 1 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + null, + null + ], + "range_flattens": [ + null, + null, + null, + null + ], + "load_eviction_policies": [ + "first", + "", + "first", + "", + "last", + "first", + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 7, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 8192 + ], + "range_unroll_factors": [ + 0, + 2, + 4, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + null, + true, + null + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "last", + "first", + "first", + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + true, + true + ], + "range_flattens": [ + null, + false, + false, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 2, + 2, + 1 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + null + ], + "range_flattens": [ + null, + false, + null, + false + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "first", + "first", + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + false + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 2, + 4, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + false + ], + "range_flattens": [ + null, + false, + false, + true + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "last", + "", + "first", + "last", + "", + "" + ], + "num_warps": 16, + "num_stages": 3, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 2, + 0, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + true, + true, + false + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "last", + "last", + "last", + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 4, + 1 + ], + "range_warp_specializes": [ + null, + false, + null, + false + ], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + false + ], + "range_flattens": [ + null, + true, + false, + true + ], + "load_eviction_policies": [ + "last", + "", + "last", + "first", + "first", + "", + "last", + "", + "last" + ], + "num_warps": 8, + "num_stages": 7, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + true, + true + ], + "range_flattens": [ + null, + false, + false, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 0, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + null + ], + "range_flattens": [ + null, + true, + null, + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 7, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 0, + 0, + 1 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + false + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "first", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + null, + true + ], + "range_flattens": [ + null, + false, + null, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "last", + "first", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 1024 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + null, + null + ], + "range_flattens": [ + null, + null, + true, + null + ], + "load_eviction_policies": [ + "first", + "", + "last", + "last", + "", + "last", + "first", + "first", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 1024 + ], + "range_unroll_factors": [ + 0, + 3, + 0, + 0 + ], + "range_warp_specializes": [ + null, + null, + null, + true + ], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + true + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "first", + "last", + "first", + "last", + "", + "last", + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 2 + ], + "range_warp_specializes": [ + null, + null, + false, + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + false, + null, + true + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "", + "last", + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 8, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 1024 + ], + "range_unroll_factors": [ + 0, + 0, + 4, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + false, + false + ], + "range_flattens": [ + null, + null, + true, + null + ], + "load_eviction_policies": [ + "first", + "", + "first", + "", + "last", + "last", + "first", + "first", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + null, + true + ], + "range_flattens": [ + null, + false, + null, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "last", + "first", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 1024 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + null, + null + ], + "range_flattens": [ + null, + null, + true, + null + ], + "load_eviction_policies": [ + "first", + "", + "last", + "last", + "", + "last", + "first", + "first", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 1024 + ], + "range_unroll_factors": [ + 0, + 1, + 4, + 2 + ], + "range_warp_specializes": [ + null, + false, + false, + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + null, + true + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "first", + "", + "last", + "last", + "", + "", + "", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 2 + ], + "range_warp_specializes": [ + null, + null, + false, + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + false, + null, + true + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "", + "last", + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 8, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + false + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 1024 + ], + "range_unroll_factors": [ + 0, + 0, + 4, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + false, + false + ], + "range_flattens": [ + null, + null, + true, + null + ], + "load_eviction_policies": [ + "first", + "", + "first", + "", + "last", + "last", + "first", + "first", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 2 + ], + "range_warp_specializes": [ + null, + null, + false, + null + ], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + false, + null, + true + ], + "load_eviction_policies": [ + "", + "last", + "", + "last", + "", + "last", + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 8, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + false + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + } +] \ No newline at end of file diff --git a/vllm/kernels/helion/configs/rms_norm_dynamic_per_token_quant/nvidia_h100.json b/vllm/kernels/helion/configs/rms_norm_dynamic_per_token_quant/nvidia_h100.json new file mode 100644 index 0000000000..a58a67acc2 --- /dev/null +++ b/vllm/kernels/helion/configs/rms_norm_dynamic_per_token_quant/nvidia_h100.json @@ -0,0 +1,3663 @@ +[ + { + "key": { + "hidden_size": 2048, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 0, + 0, + 1 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + false + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "first", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 2, + 3, + 1, + 0 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + true, + null, + false, + null + ], + "range_flattens": [ + null, + null, + false, + true + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 16, + "num_stages": 7, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "persistent_interleaved", + "num_sm_multiplier": 1, + "maxnreg": 128 + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 2, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + false + ], + "range_flattens": [ + null, + false, + true, + false + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "", + "last", + "last", + "first", + "first" + ], + "num_warps": 32, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 3, + 1 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + true, + null + ], + "range_flattens": [ + null, + false, + false, + true + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "last", + "first", + "first", + "", + "" + ], + "num_warps": 8, + "num_stages": 4, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 4, + 4, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + true, + false + ], + "range_flattens": [ + null, + true, + true, + false + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "last", + "first", + "first", + "first", + "" + ], + "num_warps": 16, + "num_stages": 3, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 8192 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + null + ], + "range_flattens": [ + null, + true, + false, + true + ], + "load_eviction_policies": [ + "last", + "first", + "last", + "first", + "first", + "", + "last", + "", + "last" + ], + "num_warps": 32, + "num_stages": 8, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 1 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + null + ], + "range_flattens": [ + null, + true, + null, + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "first", + "last", + "first", + "last" + ], + "num_warps": 8, + "num_stages": 4, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + false, + null + ], + "range_flattens": [ + null, + false, + true, + true + ], + "load_eviction_policies": [ + "", + "first", + "", + "first", + "last", + "", + "last", + "", + "last" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 4096 + ], + "range_unroll_factors": [ + 0, + 1, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + null + ], + "range_flattens": [ + null, + false, + true, + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "last", + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + true, + true + ], + "range_flattens": [ + null, + false, + false, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 0, + 1, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + null, + false + ], + "range_flattens": [ + null, + null, + false, + true + ], + "load_eviction_policies": [ + "first", + "", + "last", + "", + "first", + "last", + "last", + "last", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 8192 + ], + "range_unroll_factors": [ + 0, + 2, + 0, + 1 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + null, + false + ], + "range_flattens": [ + null, + null, + false, + null + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "", + "", + "first", + "first", + "" + ], + "num_warps": 32, + "num_stages": 7, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + null, + true + ], + "range_flattens": [ + null, + true, + true, + false + ], + "load_eviction_policies": [ + "first", + "first", + "first", + "", + "", + "last", + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 4, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 2, + 4, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + false, + true + ], + "range_flattens": [ + null, + null, + false, + true + ], + "load_eviction_policies": [ + "", + "", + "first", + "", + "last", + "last", + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 4, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 8192 + ], + "range_unroll_factors": [ + 0, + 2, + 4, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + null, + true, + null + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "last", + "first", + "first", + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 2, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + null, + null + ], + "range_flattens": [ + null, + true, + true, + false + ], + "load_eviction_policies": [ + "last", + "", + "last", + "last", + "last", + "last", + "last", + "first", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 2, + 2, + 1 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + null + ], + "range_flattens": [ + null, + false, + null, + false + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "first", + "first", + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 8192 + ], + "range_unroll_factors": [ + 0, + 3, + 3, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + false, + false + ], + "range_flattens": [ + null, + true, + null, + true + ], + "load_eviction_policies": [ + "first", + "last", + "first", + "last", + "", + "", + "last", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 2, + 4, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + false + ], + "range_flattens": [ + null, + false, + false, + true + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "last", + "", + "first", + "last", + "", + "" + ], + "num_warps": 16, + "num_stages": 3, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 2, + 0, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + true, + true, + false + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "last", + "last", + "last", + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 8192 + ], + "range_unroll_factors": [ + 0, + 2, + 2, + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + true, + null + ], + "range_flattens": [ + null, + false, + true, + null + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "", + "first", + "", + "first", + "" + ], + "num_warps": 32, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + true, + true + ], + "range_flattens": [ + null, + false, + false, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 1, + 2, + 1 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + null, + null + ], + "range_flattens": [ + null, + null, + null, + null + ], + "load_eviction_policies": [ + "first", + "", + "first", + "", + "last", + "first", + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 7, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 4096 + ], + "range_unroll_factors": [ + 0, + 2, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + false + ], + "range_flattens": [ + null, + true, + false, + true + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "last", + "last", + "", + "", + "last" + ], + "num_warps": 16, + "num_stages": 7, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 1, + 2, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + null, + true, + false + ], + "load_eviction_policies": [ + "", + "", + "first", + "", + "", + "", + "last", + "", + "" + ], + "num_warps": 8, + "num_stages": 8, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 0, + 1, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + null, + null + ], + "range_flattens": [ + null, + null, + true, + true + ], + "load_eviction_policies": [ + "first", + "", + "first", + "", + "", + "last", + "last", + "last", + "last" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 4096 + ], + "range_unroll_factors": [ + 0, + 1, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + null + ], + "range_flattens": [ + null, + false, + true, + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "last", + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + false, + false, + false + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "", + "first", + "first", + "last", + "last" + ], + "num_warps": 8, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + null, + true + ], + "range_flattens": [ + null, + false, + null, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "last", + "first", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + false + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 1024 + ], + "range_unroll_factors": [ + 0, + 0, + 4, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + false, + false + ], + "range_flattens": [ + null, + null, + true, + null + ], + "load_eviction_policies": [ + "first", + "", + "first", + "", + "last", + "last", + "first", + "first", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 0, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + null + ], + "range_flattens": [ + null, + true, + null, + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 7, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 4096 + ], + "range_unroll_factors": [ + 0, + 1, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + null + ], + "range_flattens": [ + null, + false, + true, + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "last", + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 1024 + ], + "range_unroll_factors": [ + 0, + 3, + 0, + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + null, + false, + null + ], + "load_eviction_policies": [ + "", + "", + "first", + "", + "", + "", + "first", + "", + "" + ], + "num_warps": 4, + "num_stages": 4, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + null, + true + ], + "range_flattens": [ + null, + false, + null, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "last", + "first", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 4096 + ], + "range_unroll_factors": [ + 0, + 1, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + null + ], + "range_flattens": [ + null, + false, + true, + false + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "last", + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 1024 + ], + "range_unroll_factors": [ + 0, + 4, + 1, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + null, + true, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "last", + "", + "", + "first", + "first", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 2, + 4, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + false, + null, + null + ], + "load_eviction_policies": [ + "last", + "", + "", + "last", + "first", + "", + "first", + "", + "last" + ], + "num_warps": 16, + "num_stages": 8, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 8192, + 4096, + 4096 + ], + "range_unroll_factors": [ + 1, + 1, + 2, + 1 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + null, + false, + false, + true + ], + "range_flattens": [ + true, + null, + null, + true + ], + "load_eviction_policies": [ + "", + "last", + "", + "first", + "first", + "last", + "first", + "first", + "" + ], + "num_warps": 16, + "num_stages": 8, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_interleaved", + "num_sm_multiplier": 32, + "maxnreg": 128 + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 1024 + ], + "range_unroll_factors": [ + 0, + 4, + 1, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + null + ], + "range_flattens": [ + null, + null, + true, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "last", + "", + "", + "first", + "first", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 2048 + ], + "range_unroll_factors": [ + 2, + 2, + 3, + 2 + ], + "range_warp_specializes": [], + "range_multi_buffers": [ + true, + false, + false, + null + ], + "range_flattens": [ + false, + null, + false, + false + ], + "load_eviction_policies": [ + "last", + "last", + "", + "last", + "last", + "", + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 7, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "persistent_interleaved", + "num_sm_multiplier": 64, + "maxnreg": 256 + } + }, + { + "key": { + "hidden_size": 8192, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 4096 + ], + "range_unroll_factors": [ + 0, + 3, + 1, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + true, + null + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "last", + "last", + "", + "last", + "last", + "", + "last", + "first", + "" + ], + "num_warps": 16, + "num_stages": 7, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 1 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 2, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + false, + false + ], + "range_flattens": [ + null, + false, + true, + false + ], + "load_eviction_policies": [ + "last", + "", + "last", + "", + "", + "last", + "last", + "first", + "first" + ], + "num_warps": 32, + "num_stages": 6, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 2 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 4 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 8 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 16 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + false, + false + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "", + "first", + "first", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 32 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + false + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 64 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 128 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 2, + 3 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + null, + null, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "last", + "", + "first", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 256 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + false, + false + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "", + "first", + "first", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 512 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + false, + false + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "", + "", + "first", + "first", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 1024 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 1024 + ], + "range_unroll_factors": [ + 0, + 3, + 2, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + null, + null + ], + "range_flattens": [ + null, + null, + true, + null + ], + "load_eviction_policies": [ + "first", + "", + "last", + "last", + "", + "last", + "first", + "first", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 2048 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + false + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 4096 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + false + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 8192 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 4, + 3, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + false + ], + "range_flattens": [ + null, + true, + true, + null + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 5120, + "num_tokens": 16384 + }, + "config": { + "block_sizes": [ + 8192, + 8192, + 2048 + ], + "range_unroll_factors": [ + 0, + 0, + 2, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + true, + null, + true + ], + "range_flattens": [ + null, + true, + null, + true + ], + "load_eviction_policies": [ + "last", + "", + "last", + "last", + "last", + "last", + "first", + "last", + "" + ], + "num_warps": 8, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 2048, + "num_tokens": 16384 + }, + "config": { + "block_sizes": [ + 2048, + 2048, + 2048 + ], + "range_unroll_factors": [ + 0, + 2, + 4, + 4 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + false, + null, + false + ], + "range_flattens": [ + null, + true, + null, + true + ], + "load_eviction_policies": [ + "", + "", + "", + "", + "last", + "", + "first", + "", + "first" + ], + "num_warps": 8, + "num_stages": 6, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + }, + { + "key": { + "hidden_size": 4096, + "num_tokens": 16384 + }, + "config": { + "block_sizes": [ + 4096, + 4096, + 4096 + ], + "range_unroll_factors": [ + 0, + 4, + 4, + 2 + ], + "range_warp_specializes": [], + "range_num_stages": [], + "range_multi_buffers": [ + null, + null, + true, + true + ], + "range_flattens": [ + null, + true, + null, + false + ], + "load_eviction_policies": [ + "last", + "last", + "last", + "", + "last", + "first", + "first", + "", + "last" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "atomic_indexing": [], + "pid_type": "flat" + } + } +] \ No newline at end of file diff --git a/vllm/kernels/helion/ops/rms_norm_dynamic_per_token_quant.py b/vllm/kernels/helion/ops/rms_norm_dynamic_per_token_quant.py new file mode 100644 index 0000000000..f15132c27c --- /dev/null +++ b/vllm/kernels/helion/ops/rms_norm_dynamic_per_token_quant.py @@ -0,0 +1,233 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from itertools import product +from typing import Any + +import torch + +from vllm.kernels.helion.case_key import CaseKey +from vllm.kernels.helion.utils import ( + get_fp8_dtype, + get_int8_min_max, + get_int8_min_scaling_factor, +) +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + get_fp8_min_max, +) +from vllm.platforms import current_platform +from vllm.utils.import_utils import has_helion + +if not has_helion(): + raise ImportError( + "Helion kernel requires helion to be installed. " + "Install it with: pip install helion" + ) + +import helion +import helion.language as hl + +from vllm.kernels.helion.register import register_kernel + +logger = init_logger(__name__) + + +def generate_inputs() -> dict[CaseKey, tuple[Any, ...]]: + # TODO(xiaohongchen1991): it is difficult for kernel author to cover all + # input property combination. Currently, dtypes are fixed. We need + # optimization to bucket/skip some combinations + num_tokens_list = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192] + hidden_size_list = [2048, 4096, 5120] + in_dtype: torch.dtype = torch.bfloat16 + out_dtype: torch.dtype = current_platform.fp8_dtype() + scale_dtype: torch.dtype = torch.float32 + inputs = {} + + for num_tokens, hidden_size in product(num_tokens_list, hidden_size_list): + input = torch.randn(num_tokens, hidden_size, device="cuda", dtype=in_dtype) + result = torch.empty(input.shape, device=input.device, dtype=out_dtype) + scale = torch.empty((num_tokens, 1), device=input.device, dtype=scale_dtype) + scale_ub = torch.mean(input).to(scale_dtype) + residual = torch.randn_like(input) + weight = torch.normal( + mean=1.0, + std=1.0, + size=(hidden_size,), + dtype=input.dtype, + device=input.device, + ) + epsilon = 1e-6 + + config_key = CaseKey({"hidden_size": hidden_size, "num_tokens": num_tokens}) + inputs[config_key] = (result, input, weight, scale, epsilon, scale_ub, residual) + + return inputs + + +_pick_cache: dict[tuple[int, int], CaseKey | None] = {} + + +def pick_config(args: tuple[Any, ...], config_keys: list[CaseKey]) -> CaseKey | None: + """Pick the best pre-tuned config for the given input shape. + + Selection strategy: + 1. Find the closest hidden_size among available configs + (exact match preferred). + 2. Among the num_tokens values tuned for that hidden_size, pick + the smallest num_tokens >= the input's num_tokens. If the input is + larger than all available num_tokens, fall back to the largest. + """ + + if not config_keys: + return None + + _, input, *_ = args + num_tokens, hidden_size = input.shape + + cache_key = (num_tokens, hidden_size) + cached = _pick_cache.get(cache_key) + if cached is not None: + return cached + + configs: dict[int, list[int]] = {} + for key in config_keys: + if key.is_default(): + continue + configs.setdefault(key["hidden_size"], []).append(key["num_tokens"]) + + if not configs: + return None + + best_hidden_size = min(configs, key=lambda s: abs(s - hidden_size)) + available_num_tokens = sorted(configs[best_hidden_size]) + best_num_tokens = next( + (n for n in available_num_tokens if n >= num_tokens), available_num_tokens[-1] + ) + + result = CaseKey({"hidden_size": best_hidden_size, "num_tokens": best_num_tokens}) + _pick_cache[cache_key] = result + return result + + +def fake_impl( + result: torch.Tensor, # [num_tokens, hidden_size] + input: torch.Tensor, # [num_tokens, hidden_size] + weight: torch.Tensor, # [hidden_size] + scale: torch.Tensor, # [num_tokens, 1] + epsilon: float, + scale_ub: torch.Tensor | None = None, # [] + residual: torch.Tensor | None = None, # [num_tokens, hidden_size] +) -> None: + return + + +def baseline( + result: torch.Tensor, # [num_tokens, hidden_size] + input: torch.Tensor, # [num_tokens, hidden_size] + weight: torch.Tensor, # [num_tokens] + scale: torch.Tensor, # [num_tokens, 1] + epsilon: float, + scale_ub: torch.Tensor | None = None, # [] + residual: torch.Tensor | None = None, # [num_tokens, hidden_size] +) -> None: + torch.ops._C.rms_norm_dynamic_per_token_quant( + result, input, weight, scale, epsilon, scale_ub, residual + ) + + +# Overwrite autotune_baseline_atol and autotune_baseline_rtol +# if too many configs failed due to baseline check during autotuning +@register_kernel( + mutates_args=["result", "scale", "residual"], + config_picker=pick_config, + input_generator=generate_inputs, + fake_impl=fake_impl, + helion_settings=helion.Settings( + autotune_baseline_fn=baseline, + ignore_warnings=[helion.exc.TensorOperationInWrapper], + ), +) # type: ignore[misc] +def rms_norm_dynamic_per_token_quant( + result: torch.Tensor, # [num_tokens, hidden_size] + input: torch.Tensor, # [num_tokens, hidden_size] + weight: torch.Tensor, # [hidden_size] + scale: torch.Tensor, # [num_tokens, 1] + epsilon: float, + scale_ub: torch.Tensor | None = None, # [] + residual: torch.Tensor | None = None, # [num_tokens, hidden_size] +) -> None: + # This code assumes batch_dim and num_tokens are flattened + assert input.ndim == 2 + num_tokens, hidden_size = input.shape + hl.specialize(hidden_size) + + fp8_dtype = get_fp8_dtype() + assert result.dtype in [fp8_dtype, torch.int8] + assert result.is_contiguous() and input.is_contiguous() + + if scale_ub is not None: + assert result.dtype == fp8_dtype + assert scale_ub.dtype == torch.float32 + + assert input.dtype == weight.dtype + assert scale.shape[0] == num_tokens + assert scale.dtype == torch.float32 + + if residual is not None: + assert residual.dtype == input.dtype + + quant_dtype = result.dtype + qtype_traits_min: int | float + qtype_traits_max: int | float + if quant_dtype == torch.int8: + qtype_traits_min, qtype_traits_max = get_int8_min_max() + min_scaling_factor = get_int8_min_scaling_factor() + else: + qtype_traits_min, qtype_traits_max = get_fp8_min_max() + min_scaling_factor = 1.0 / (qtype_traits_max * 512.0) + + qtype_max = float(qtype_traits_max) + + for tile_m in hl.tile(num_tokens, block_size=1): + rms = hl.zeros([tile_m], dtype=torch.float32) + for tile_n in hl.tile(hidden_size): + x_blk = input[tile_m, tile_n].to(torch.float32) + if residual is not None: + x_blk = x_blk + residual[tile_m, tile_n] + rms = rms + x_blk.pow(2).sum(dim=-1) + + rms = torch.rsqrt(rms * (1.0 / hidden_size) + epsilon) + s_blk = hl.zeros([tile_m], dtype=torch.float32) + + for tile_n in hl.tile(hidden_size): + x_blk = input[tile_m, tile_n].to(torch.float32) + if residual is not None: + x_blk = x_blk + residual[tile_m, tile_n] + x_blk = (x_blk * rms[:, None]).to(input.dtype) * weight[None, tile_n] + tmp_blk = torch.amax(torch.abs(x_blk), dim=-1).to(torch.float32) + s_blk = torch.maximum(s_blk, tmp_blk) + + if scale_ub is not None: + scale_ub_s = hl.load(scale_ub, []) + s_blk = s_blk.clamp(max=scale_ub_s) + s_blk = s_blk * (1.0 / qtype_max) + s_blk = s_blk.clamp(min=min_scaling_factor) + scale[tile_m, 0] = s_blk + + for tile_n in hl.tile(hidden_size): + x_blk = input[tile_m, tile_n].to(torch.float32) + if residual is not None: + x_blk = x_blk + residual[tile_m, tile_n] + residual[tile_m, tile_n] = x_blk.to(residual.dtype) + x_blk = (x_blk * rms[:, None]).to(input.dtype) * weight[None, tile_n] + if quant_dtype == torch.int8: + s_inv_blk = 1.0 / s_blk[:, None] + y_blk = x_blk * s_inv_blk + y_blk = y_blk.round() + else: + y_blk = x_blk / s_blk[:, None] + + result[tile_m, tile_n] = y_blk.clamp(qtype_traits_min, qtype_traits_max).to( + result.dtype + )