[Kernel] add Flashinfer cutedsl w4a16 linear (#53014)

Signed-off-by: Siyuan Fu <siyuanf@nvidia.com>
This commit is contained in:
Lain
2026-09-01 02:49:14 +00:00
committed by GitHub
parent 907b1a7f22
commit 882ca8d696
8 changed files with 230 additions and 10 deletions
@@ -20,14 +20,94 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
flashinfer_prepare_bf16_fp4_weights,
flashinfer_scaled_fp4_mm,
has_flashinfer,
has_flashinfer_b12x_gemm,
has_flashinfer_bf16_fp4,
)
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
class FlashInferCuteDslNvFp4W4A16LinearKernel(NvFp4LinearKernel):
"""BF16 x NVFP4 GEMM via FlashInfer's CuTe-DSL backend."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if compute_capability is None:
if not current_platform.is_cuda():
return False, "FlashInfer CuTe-DSL W4A16 requires CUDA"
capability = current_platform.get_device_capability()
if capability is None:
return False, "CUDA compute capability is unavailable"
compute_capability = capability.to_int()
if compute_capability not in (100, 103) and not (
compute_capability >= 120 and compute_capability < 130
):
return False, "FlashInfer CuTe-DSL W4A16 requires sm_100 or sm_12x"
if not has_flashinfer_bf16_fp4():
return False, "FlashInfer CuTe-DSL BF16 x FP4 GEMM is unavailable"
return True, None
@classmethod
def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
padded_weight, weights_padding_bytes = pad_nvfp4_weight_for_cutlass(
layer.weight.data, alignment=64
)
swizzled_scale = swizzle_blockscale(layer.weight_scale.data)
weight, weight_scale, global_scale = flashinfer_prepare_bf16_fp4_weights(
padded_weight,
swizzled_scale,
layer.weight_global_scale.data.reshape(1),
backend="cute-dsl",
)
assert global_scale is not None
layer.weight = torch.nn.Parameter(weight, requires_grad=False)
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
layer.weight_global_scale = torch.nn.Parameter(
global_scale, requires_grad=False
)
layer.weights_padding_cols = weights_padding_bytes
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if x.dtype != torch.bfloat16:
raise ValueError(
f"FlashInfer CuTe-DSL W4A16 requires BF16 input, got {x.dtype}"
)
output_size = layer.output_size_per_partition
output_shape = [*x.shape[:-1], output_size]
x_2d = x.reshape(-1, x.shape[-1])
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
if weights_padding_bytes:
x_2d = torch.nn.functional.pad(x_2d, (0, weights_padding_bytes * 2))
x_2d = x_2d.contiguous()
out = torch.ops.vllm.flashinfer_mm_bf16_fp4(
x_2d,
layer.weight,
layer.weight_scale,
layer.weight_global_scale,
)
out = slice_nvfp4_output(out, output_size)
if bias is not None:
out = out + bias
return out.view(*output_shape)
class FlashInferCuteDslNvFp4LinearKernel(NvFp4LinearKernel):
"""NVFP4 GEMM via FlashInfer's cutedsl backend."""