[Kernel] add Flashinfer cutedsl w4a16 linear (#53014)

Signed-off-by: Siyuan Fu <siyuanf@nvidia.com>
This commit is contained in:
Lain
2026-09-01 02:49:14 +00:00
committed by GitHub
parent 907b1a7f22
commit 882ca8d696
8 changed files with 230 additions and 10 deletions
+49
View File
@@ -197,6 +197,9 @@ flashinfer_cutlass_fused_moe = _lazy_import_wrapper(
flashinfer_cutedsl_grouped_gemm_nt_masked = _lazy_import_wrapper(
"flashinfer.cute_dsl.blockscaled_gemm", "grouped_gemm_nt_masked"
)
flashinfer_prepare_bf16_fp4_weights = _lazy_import_wrapper(
"flashinfer.gemm", "prepare_bf16_fp4_weights"
)
flashinfer_fp4_quantize = _lazy_import_wrapper("flashinfer", "fp4_quantize")
flashinfer_mxfp4_quantize = _lazy_import_wrapper("flashinfer", "mxfp4_quantize")
nvfp4_batched_quantize = _lazy_import_wrapper("flashinfer", "nvfp4_batched_quantize")
@@ -333,6 +336,17 @@ def has_flashinfer_cutedsl() -> bool:
)
@functools.cache
def has_flashinfer_bf16_fp4() -> bool:
"""Return ``True`` if FlashInfer's CuTe-DSL W4A16 GEMM is available."""
if not has_flashinfer_cutedsl():
return False
mod = _get_submodule("flashinfer.gemm")
return mod is not None and all(
hasattr(mod, name) for name in ("mm_bf16_fp4", "prepare_bf16_fp4_weights")
)
@functools.cache
def has_flashinfer_trtllm_fused_moe() -> bool:
"""Return `True` if FlashInfer TRTLLM fused MoE is available."""
@@ -706,6 +720,39 @@ if has_flashinfer():
) -> torch.Tensor:
return torch.empty(A.shape[0], B.shape[1], dtype=dtype, device=A.device)
@torch.library.custom_op(
"vllm::flashinfer_mm_bf16_fp4",
mutates_args=[],
device_types="cuda",
)
def flashinfer_mm_bf16_fp4(
A: torch.Tensor,
B: torch.Tensor,
B_scale: torch.Tensor,
global_scale: torch.Tensor,
) -> torch.Tensor:
from flashinfer.gemm import mm_bf16_fp4
return mm_bf16_fp4(
A,
B,
B_scale,
global_scale,
backend="cute-dsl",
)
@torch.library.register_fake(
"vllm::flashinfer_mm_bf16_fp4",
)
def flashinfer_mm_bf16_fp4_fake(
A: torch.Tensor,
B: torch.Tensor,
B_scale: torch.Tensor,
global_scale: torch.Tensor,
) -> torch.Tensor:
output_size = B.shape[0] if B.dtype == torch.uint8 else B.shape[1] // 2
return torch.empty(A.shape[0], output_size, dtype=A.dtype, device=A.device)
@torch.library.custom_op(
"vllm::flashinfer_mxfp4_quantize",
mutates_args=[],
@@ -1158,6 +1205,7 @@ __all__ = [
"flashinfer_trtllm_fp8_block_scale_moe",
"flashinfer_cutlass_fused_moe",
"flashinfer_cutedsl_grouped_gemm_nt_masked",
"flashinfer_prepare_bf16_fp4_weights",
"flashinfer_fp4_quantize",
"silu_and_mul_scaled_nvfp4_experts_quantize",
"scaled_fp4_grouped_quantize",
@@ -1177,6 +1225,7 @@ __all__ = [
"has_flashinfer_cutlass_fused_moe",
"has_flashinfer_cutedsl_grouped_gemm_nt_masked",
"has_flashinfer_cutedsl_moe_nvfp4",
"has_flashinfer_bf16_fp4",
"has_flashinfer_b12x_moe",
"has_flashinfer_b12x_gemm",
"has_flashinfer_fp8_blockscale_gemm",