[Performance] Use CuTe-DSL for FlashInfer MXFP4 quantization (#48417)
Signed-off-by: BWAAEEEK <jooho414@gmail.com>
This commit is contained in:
@@ -56,7 +56,9 @@ class FlashInferMxFp4LinearKernel(MxFp4LinearKernel):
|
|||||||
out_shape = x.shape[:-1] + (layer.output_size_per_partition,)
|
out_shape = x.shape[:-1] + (layer.output_size_per_partition,)
|
||||||
x_2d = x.reshape(-1, x.shape[-1])
|
x_2d = x.reshape(-1, x.shape[-1])
|
||||||
|
|
||||||
x_fp4, x_scale = flashinfer_mxfp4_quantize(x_2d.contiguous())
|
x_fp4, x_scale = flashinfer_mxfp4_quantize(
|
||||||
|
x_2d.contiguous(), backend="cute-dsl"
|
||||||
|
)
|
||||||
out = flashinfer_scaled_fp4_mm(
|
out = flashinfer_scaled_fp4_mm(
|
||||||
x_fp4,
|
x_fp4,
|
||||||
weight,
|
weight,
|
||||||
|
|||||||
@@ -618,14 +618,16 @@ if has_flashinfer():
|
|||||||
)
|
)
|
||||||
def flashinfer_mxfp4_quantize(
|
def flashinfer_mxfp4_quantize(
|
||||||
a: torch.Tensor,
|
a: torch.Tensor,
|
||||||
|
backend: str,
|
||||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
from flashinfer import mxfp4_quantize as _mxfp4_quantize
|
from flashinfer import mxfp4_quantize as _mxfp4_quantize
|
||||||
|
|
||||||
return _mxfp4_quantize(a)
|
return _mxfp4_quantize(a, backend=backend)
|
||||||
|
|
||||||
@torch.library.register_fake("vllm::flashinfer_mxfp4_quantize")
|
@torch.library.register_fake("vllm::flashinfer_mxfp4_quantize")
|
||||||
def flashinfer_mxfp4_quantize_fake(
|
def flashinfer_mxfp4_quantize_fake(
|
||||||
a: torch.Tensor,
|
a: torch.Tensor,
|
||||||
|
backend: str,
|
||||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
m, k = a.shape
|
m, k = a.shape
|
||||||
sf_vec_size = 32
|
sf_vec_size = 32
|
||||||
|
|||||||
Reference in New Issue
Block a user