[Kernel] Add nvfp4 gemm flashinfer backends (#22346)
Signed-off-by: Julien Lin <jullin@nvidia.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -14,6 +14,7 @@ import os
|
||||
from typing import Any, Callable, NoReturn, Optional
|
||||
|
||||
import requests
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
@@ -193,6 +194,75 @@ def use_trtllm_attention(
|
||||
return use_trtllm
|
||||
|
||||
|
||||
if has_flashinfer():
|
||||
|
||||
@torch.library.custom_op(
|
||||
"vllm::flashinfer_mm_fp4",
|
||||
mutates_args=[],
|
||||
device_types="cuda",
|
||||
)
|
||||
def flashinfer_mm_fp4(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
A_scale: torch.Tensor,
|
||||
B_scale: torch.Tensor,
|
||||
g_scale: torch.Tensor,
|
||||
dtype: torch.dtype,
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
from flashinfer import mm_fp4 as flashinfer_mm_fp4_
|
||||
return flashinfer_mm_fp4_(A,
|
||||
B,
|
||||
A_scale,
|
||||
B_scale,
|
||||
g_scale,
|
||||
dtype,
|
||||
block_size=16,
|
||||
backend=backend)
|
||||
|
||||
@torch.library.register_fake("vllm::flashinfer_mm_fp4", )
|
||||
def flashinfer_mm_fp4_fake(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
A_scale: torch.Tensor,
|
||||
B_scale: torch.Tensor,
|
||||
g_scale: torch.Tensor,
|
||||
dtype: torch.dtype,
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(A.shape[0],
|
||||
B.shape[1],
|
||||
dtype=dtype,
|
||||
device=A.device)
|
||||
|
||||
|
||||
def flashinfer_scaled_fp4_mm(a: torch.Tensor, b: torch.Tensor,
|
||||
block_scale_a: torch.Tensor,
|
||||
block_scale_b: torch.Tensor, alpha: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
backend: str) -> torch.Tensor:
|
||||
assert a.ndim == 2 and b.ndim == 2
|
||||
assert block_scale_a.ndim == 2 and block_scale_b.ndim == 2
|
||||
assert a.stride(-1) == 1 and b.stride(-1) == 1
|
||||
assert a.shape[1] == b.shape[1]
|
||||
assert block_scale_a.shape[1] == a.shape[1] // 8
|
||||
assert block_scale_b.shape[1] == b.shape[1] // 8
|
||||
|
||||
if backend == "cutlass":
|
||||
block_scale_a = block_scale_a.view(torch.uint8)
|
||||
block_scale_b = block_scale_b.view(torch.uint8)
|
||||
|
||||
return flashinfer_mm_fp4(
|
||||
a,
|
||||
b.t(),
|
||||
block_scale_a,
|
||||
block_scale_b.t(),
|
||||
alpha,
|
||||
out_dtype,
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"has_flashinfer",
|
||||
"flashinfer_trtllm_fp8_block_scale_moe",
|
||||
@@ -205,4 +275,5 @@ __all__ = [
|
||||
"has_flashinfer_cutlass_fused_moe",
|
||||
"has_nvidia_artifactory",
|
||||
"use_trtllm_attention",
|
||||
"flashinfer_scaled_fp4_mm",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user