[Kernel] add Flashinfer cutedsl w4a16 linear (#53014)
Signed-off-by: Siyuan Fu <siyuanf@nvidia.com>
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from nvfp4_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
FLOAT8_E4M3_MAX,
|
||||
@@ -10,6 +11,13 @@ from nvfp4_utils import (
|
||||
)
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.kernels.linear.nvfp4 import NvFp4LinearLayerConfig
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_to_dtype,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_scaled_fp4_mm,
|
||||
@@ -166,3 +174,55 @@ def test_flashinfer_nvfp4_gemm(
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, expected_out.to(dtype=dtype), atol=1e-1, rtol=1e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", SHAPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_cutedsl_nvfp4_w4a16_linear(
|
||||
shape: tuple[int, int, int],
|
||||
seed: int,
|
||||
device: str,
|
||||
) -> None:
|
||||
supported, reason = FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
|
||||
set_random_seed(seed)
|
||||
m, n, packed_k = shape
|
||||
k = packed_k * 2
|
||||
x = torch.randn((m, k), dtype=torch.bfloat16, device=device)
|
||||
weight = torch.randn((n, k), dtype=torch.bfloat16, device=device)
|
||||
weight_quant_scale = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / weight.abs().max()).to(
|
||||
torch.float32
|
||||
)
|
||||
weight_global_scale = weight_quant_scale.reciprocal()
|
||||
weight_fp4, weight_scale = ops.scaled_fp4_quant(
|
||||
weight,
|
||||
weight_quant_scale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
weight_ref = dequantize_to_dtype(
|
||||
weight_fp4,
|
||||
weight_scale,
|
||||
weight_global_scale,
|
||||
dtype=torch.bfloat16,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.output_size_per_partition = n
|
||||
layer.weight = torch.nn.Parameter(weight_fp4, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
|
||||
layer.weight_global_scale = torch.nn.Parameter(
|
||||
weight_global_scale, requires_grad=False
|
||||
)
|
||||
kernel = FlashInferCuteDslNvFp4W4A16LinearKernel(NvFp4LinearLayerConfig())
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
output = kernel.apply_weights(layer, x)
|
||||
|
||||
expected = F.linear(x, weight_ref)
|
||||
assert output.shape == (m, n)
|
||||
torch.testing.assert_close(output, expected, atol=1e-1, rtol=1e-1)
|
||||
|
||||
@@ -16,6 +16,7 @@ from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
HummingNvFp4LinearKernel,
|
||||
MarlinNvFp4LinearKernel,
|
||||
)
|
||||
@@ -584,10 +585,19 @@ def test_modelopt_nvfp4_config_dispatches_w4a16_method():
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("linear_backend", "kernel_cls"),
|
||||
[("auto", MarlinNvFp4LinearKernel), ("humming", HummingNvFp4LinearKernel)],
|
||||
[
|
||||
("auto", MarlinNvFp4LinearKernel),
|
||||
("humming", HummingNvFp4LinearKernel),
|
||||
("flashinfer_cutedsl", FlashInferCuteDslNvFp4W4A16LinearKernel),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
def test_modelopt_w4a16_respects_linear_backend(linear_backend, kernel_cls):
|
||||
if linear_backend != "auto":
|
||||
is_supported, reason = kernel_cls.is_supported()
|
||||
if not is_supported:
|
||||
pytest.skip(reason)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.kernel_config.linear_backend = linear_backend
|
||||
with set_current_vllm_config(vllm_config):
|
||||
|
||||
Reference in New Issue
Block a user