Integrate flashinfer b12x MoE and FP4 GEMM kernels for SM120/121 (#40082)
Signed-off-by: Meenakshi Venkataraman <meenakshiv@nvidia.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
f2d5e3d3ae
commit
363fc84407
@@ -102,6 +102,7 @@ from vllm.model_executor.kernels.linear.nvfp4.fbgemm import (
|
||||
FbgemmNvFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferB12xNvFp4LinearKernel,
|
||||
FlashInferCudnnNvFp4LinearKernel,
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
FlashInferTrtllmNvFp4LinearKernel,
|
||||
@@ -367,6 +368,9 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
|
||||
_POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
|
||||
PlatformEnum.CUDA: [
|
||||
# FlashInferB12xNvFp4LinearKernel excluded from auto-selection until
|
||||
# upstream CUTLASS SM121 MMA op guard is resolved; use
|
||||
# VLLM_NVFP4_GEMM_BACKEND=flashinfer-b12x to opt in explicitly.
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
CutlassNvFp4LinearKernel,
|
||||
MarlinNvFp4LinearKernel,
|
||||
@@ -812,6 +816,7 @@ def init_wfp8_a16_linear_kernel(
|
||||
|
||||
# Maps VLLM_NVFP4_GEMM_BACKEND env var values to kernel classes.
|
||||
_NVFP4_BACKEND_TO_KERNEL: dict[str, type[NvFp4LinearKernel]] = {
|
||||
"flashinfer-b12x": FlashInferB12xNvFp4LinearKernel,
|
||||
"flashinfer-cutlass": FlashInferCutlassNvFp4LinearKernel,
|
||||
"cutlass": CutlassNvFp4LinearKernel,
|
||||
"marlin": MarlinNvFp4LinearKernel,
|
||||
@@ -1041,6 +1046,7 @@ __all__ = [
|
||||
"CutlassNvFp4LinearKernel",
|
||||
"EmulationNvFp4LinearKernel",
|
||||
"FbgemmNvFp4LinearKernel",
|
||||
"FlashInferB12xNvFp4LinearKernel",
|
||||
"FlashInferCutlassNvFp4LinearKernel",
|
||||
"FlashInferTrtllmNvFp4LinearKernel",
|
||||
"FlashInferCudnnNvFp4LinearKernel",
|
||||
|
||||
@@ -10,7 +10,11 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
swizzle_blockscale,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import flashinfer_scaled_fp4_mm, has_flashinfer
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_scaled_fp4_mm,
|
||||
has_flashinfer,
|
||||
has_flashinfer_b12x_gemm,
|
||||
)
|
||||
|
||||
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
|
||||
|
||||
@@ -211,3 +215,70 @@ class FlashInferCudnnNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
return out.view(*output_shape)
|
||||
|
||||
|
||||
class FlashInferB12xNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
"""NVFP4 GEMM via FlashInfer's b12x CuTe DSL warp-level MMA kernel (SM120+)."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if current_platform.has_device_capability(120) and has_flashinfer_b12x_gemm():
|
||||
return True, None
|
||||
return (
|
||||
False,
|
||||
"FlashInfer b12x requires SM120+ and FlashInfer "
|
||||
"with Sm120BlockScaledDenseGemmKernel",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
swizzle_blockscale(layer.weight_scale.data), requires_grad=False
|
||||
)
|
||||
padded_weight, weights_padding_cols = pad_nvfp4_weight_for_cutlass(
|
||||
layer.weight.data
|
||||
)
|
||||
layer.weight = torch.nn.Parameter(padded_weight, requires_grad=False)
|
||||
layer.weights_padding_cols = weights_padding_cols
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="b12x",
|
||||
)
|
||||
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4, getattr(layer, "weights_padding_cols", 0)
|
||||
)
|
||||
|
||||
out = flashinfer_scaled_fp4_mm(
|
||||
x_fp4,
|
||||
layer.weight,
|
||||
x_blockscale,
|
||||
layer.weight_scale,
|
||||
layer.alpha,
|
||||
output_dtype,
|
||||
backend="b12x",
|
||||
)
|
||||
|
||||
out = slice_nvfp4_output(out, output_size)
|
||||
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
return out.view(*output_shape)
|
||||
|
||||
Reference in New Issue
Block a user