[Perf] Padded nvfp4 quant kernel to remove additional copy, 2.4%~5.7% e2e performance improvement (#42774)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-05-18 16:38:12 -07:00
committed by GitHub
parent 57fef4e0bf
commit 37ece593c1
5 changed files with 119 additions and 43 deletions
@@ -9,7 +9,6 @@ from vllm._custom_ops import (
)
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
cutlass_fp4_supported,
pad_nvfp4_activation_for_cutlass,
pad_nvfp4_weight_for_cutlass,
slice_nvfp4_output,
swizzle_blockscale,
@@ -52,16 +51,14 @@ class CutlassNvFp4LinearKernel(NvFp4LinearKernel):
output_size = layer.output_size_per_partition
output_dtype = x.dtype
output_shape = [*x.shape[:-1], output_size]
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
x_fp4, x_blockscale = scaled_fp4_quant(
x,
layer.input_global_scale_inv,
is_sf_swizzled_layout=True,
backend="cutlass",
)
x_fp4 = pad_nvfp4_activation_for_cutlass(
x_fp4, getattr(layer, "weights_padding_cols", 0)
padded_n=x.shape[-1] + weights_padding_bytes * 2,
)
out = cutlass_scaled_fp4_mm(
@@ -5,7 +5,6 @@ import torch
from vllm._custom_ops import scaled_fp4_quant
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
pad_nvfp4_activation_for_cutlass,
pad_nvfp4_weight_for_cutlass,
slice_nvfp4_output,
swizzle_blockscale,
@@ -58,16 +57,14 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
output_size = layer.output_size_per_partition
output_dtype = x.dtype
output_shape = [*x.shape[:-1], output_size]
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
x_fp4, x_blockscale = scaled_fp4_quant(
x,
layer.input_global_scale_inv,
is_sf_swizzled_layout=True,
backend="flashinfer-cutlass",
)
x_fp4 = pad_nvfp4_activation_for_cutlass(
x_fp4, getattr(layer, "weights_padding_cols", 0)
padded_n=x.shape[-1] + weights_padding_bytes * 2,
)
out = flashinfer_scaled_fp4_mm(
@@ -189,16 +186,14 @@ class FlashInferCudnnNvFp4LinearKernel(NvFp4LinearKernel):
output_size = layer.output_size_per_partition
output_dtype = x.dtype
output_shape = [*x.shape[:-1], output_size]
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
x_fp4, x_blockscale = scaled_fp4_quant(
x,
layer.input_global_scale_inv,
is_sf_swizzled_layout=True,
backend="flashinfer-cudnn",
)
x_fp4 = pad_nvfp4_activation_for_cutlass(
x_fp4, getattr(layer, "weights_padding_cols", 0)
padded_n=x.shape[-1] + weights_padding_bytes * 2,
)
out = flashinfer_scaled_fp4_mm(