[Kernel] add Flashinfer cutedsl w4a16 linear (#53014)
Signed-off-by: Siyuan Fu <siyuanf@nvidia.com>
This commit is contained in:
@@ -149,6 +149,7 @@ from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferB12xNvFp4LinearKernel,
|
||||
FlashInferCudnnNvFp4LinearKernel,
|
||||
FlashInferCuteDslNvFp4LinearKernel,
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
FlashInferTrtllmNvFp4LinearKernel,
|
||||
)
|
||||
@@ -262,6 +263,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
},
|
||||
"flashinfer_cutedsl": {
|
||||
FlashInferCuteDslNvFp4LinearKernel,
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
FlashInferCutedslMxfp8LinearKernel,
|
||||
},
|
||||
"flashinfer_trtllm": {
|
||||
@@ -531,6 +533,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
_POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
|
||||
PlatformEnum.CUDA: [
|
||||
FlashInferCuteDslNvFp4LinearKernel,
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
FlashInferB12xNvFp4LinearKernel,
|
||||
CutlassNvFp4LinearKernel,
|
||||
@@ -1014,7 +1017,11 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||
"""Select and instantiate the best NVFP4 linear kernel for the
|
||||
current platform."""
|
||||
config = NvFp4LinearLayerConfig()
|
||||
a16_kernels = (MarlinNvFp4LinearKernel, HummingNvFp4LinearKernel)
|
||||
a16_kernels = (
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
MarlinNvFp4LinearKernel,
|
||||
HummingNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
# VLLM_BATCH_INVARIANT forces deterministic execution. Prefer the
|
||||
# batch-invariant CUTLASS implementation when available, otherwise fall
|
||||
@@ -1051,8 +1058,17 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||
)
|
||||
force_kernel = EmulationNvFp4LinearKernel
|
||||
elif linear_backend == "auto" and use_a16:
|
||||
# Force a16 (Marlin) when running weight-only quantization.
|
||||
force_kernel = MarlinNvFp4LinearKernel
|
||||
_cc = current_platform.get_device_capability()
|
||||
compute_capability = _cc.to_int() if _cc is not None else None
|
||||
# Weight-only: prefer FlashInfer CuTe-DSL W4A16 on SM100/103,
|
||||
# otherwise Marlin.
|
||||
cutedsl_ok, _ = FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported(
|
||||
compute_capability
|
||||
)
|
||||
if compute_capability in (100, 103) and cutedsl_ok:
|
||||
force_kernel = FlashInferCuteDslNvFp4W4A16LinearKernel
|
||||
else:
|
||||
force_kernel = MarlinNvFp4LinearKernel
|
||||
|
||||
if force_kernel is not None:
|
||||
if use_a16 and force_kernel not in a16_kernels:
|
||||
@@ -1232,6 +1248,7 @@ __all__ = [
|
||||
"EmulationNvFp4LinearKernel",
|
||||
"FbgemmNvFp4LinearKernel",
|
||||
"FlashInferCuteDslNvFp4LinearKernel",
|
||||
"FlashInferCuteDslNvFp4W4A16LinearKernel",
|
||||
"FlashInferB12xNvFp4LinearKernel",
|
||||
"FlashInferCutlassNvFp4LinearKernel",
|
||||
"FlashInferTrtllmNvFp4LinearKernel",
|
||||
|
||||
@@ -20,14 +20,94 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_prepare_bf16_fp4_weights,
|
||||
flashinfer_scaled_fp4_mm,
|
||||
has_flashinfer,
|
||||
has_flashinfer_b12x_gemm,
|
||||
has_flashinfer_bf16_fp4,
|
||||
)
|
||||
|
||||
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
|
||||
|
||||
|
||||
class FlashInferCuteDslNvFp4W4A16LinearKernel(NvFp4LinearKernel):
|
||||
"""BF16 x NVFP4 GEMM via FlashInfer's CuTe-DSL backend."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if compute_capability is None:
|
||||
if not current_platform.is_cuda():
|
||||
return False, "FlashInfer CuTe-DSL W4A16 requires CUDA"
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None:
|
||||
return False, "CUDA compute capability is unavailable"
|
||||
compute_capability = capability.to_int()
|
||||
|
||||
if compute_capability not in (100, 103) and not (
|
||||
compute_capability >= 120 and compute_capability < 130
|
||||
):
|
||||
return False, "FlashInfer CuTe-DSL W4A16 requires sm_100 or sm_12x"
|
||||
if not has_flashinfer_bf16_fp4():
|
||||
return False, "FlashInfer CuTe-DSL BF16 x FP4 GEMM is unavailable"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
padded_weight, weights_padding_bytes = pad_nvfp4_weight_for_cutlass(
|
||||
layer.weight.data, alignment=64
|
||||
)
|
||||
swizzled_scale = swizzle_blockscale(layer.weight_scale.data)
|
||||
weight, weight_scale, global_scale = flashinfer_prepare_bf16_fp4_weights(
|
||||
padded_weight,
|
||||
swizzled_scale,
|
||||
layer.weight_global_scale.data.reshape(1),
|
||||
backend="cute-dsl",
|
||||
)
|
||||
assert global_scale is not None
|
||||
|
||||
layer.weight = torch.nn.Parameter(weight, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
|
||||
layer.weight_global_scale = torch.nn.Parameter(
|
||||
global_scale, requires_grad=False
|
||||
)
|
||||
layer.weights_padding_cols = weights_padding_bytes
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if x.dtype != torch.bfloat16:
|
||||
raise ValueError(
|
||||
f"FlashInfer CuTe-DSL W4A16 requires BF16 input, got {x.dtype}"
|
||||
)
|
||||
|
||||
output_size = layer.output_size_per_partition
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
if weights_padding_bytes:
|
||||
x_2d = torch.nn.functional.pad(x_2d, (0, weights_padding_bytes * 2))
|
||||
x_2d = x_2d.contiguous()
|
||||
|
||||
out = torch.ops.vllm.flashinfer_mm_bf16_fp4(
|
||||
x_2d,
|
||||
layer.weight,
|
||||
layer.weight_scale,
|
||||
layer.weight_global_scale,
|
||||
)
|
||||
out = slice_nvfp4_output(out, output_size)
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
return out.view(*output_shape)
|
||||
|
||||
|
||||
class FlashInferCuteDslNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
"""NVFP4 GEMM via FlashInfer's cutedsl backend."""
|
||||
|
||||
|
||||
@@ -1038,7 +1038,7 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
|
||||
|
||||
# Select LinearMethod implementation based on quant_algo (FP8 pattern).
|
||||
# NVFP4 -> W4A4: cutlass NVFP4 GEMM with input quantization
|
||||
# W4A16_NVFP4 -> W4A16: FP4 Marlin GEMM with bf16/fp16 activations
|
||||
# W4A16_NVFP4 -> W4A16: flashinfer cute-dsl or Marlin with 16-bit inputs
|
||||
if quant_method == "NVFP4":
|
||||
self.LinearMethodCls = ModelOptNvFp4LinearMethod
|
||||
elif quant_method == "W4A16_NVFP4":
|
||||
@@ -1283,7 +1283,7 @@ class ModelOptNvFp4W4A16LinearMethod(LinearMethodBase):
|
||||
self.quant_config = quant_config
|
||||
self.marlin_input_dtype = None
|
||||
# `init_nvfp4_linear_kernel(use_a16=True)` is best of both worlds:
|
||||
# 1. `use_a16=True` forces `Marlin`: https://github.com/vllm-project/vllm/commit/e68988a#diff-7135ab92aa94dfacb1ad3c77fc13f9c4ffe0b977f8eac5d86c2afe243e5f92a6R842-R889
|
||||
# 1. `use_a16=True` forces flashinfer cutedsl for sm100 and `Marlin` for others.
|
||||
# for `--linear-backend=auto`, avoiding a W4A4 kernel that requires input_scale.
|
||||
# 2. Specifying e.g. `--linear-backend=humming` will override.
|
||||
self.kernel = init_nvfp4_linear_kernel(use_a16=True)
|
||||
|
||||
Reference in New Issue
Block a user