[Perf] Padded nvfp4 quant kernel to remove additional copy, 2.4%~5.7% e2e performance improvement (#42774)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -9,7 +9,6 @@ from vllm._custom_ops import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
cutlass_fp4_supported,
|
||||
pad_nvfp4_activation_for_cutlass,
|
||||
pad_nvfp4_weight_for_cutlass,
|
||||
slice_nvfp4_output,
|
||||
swizzle_blockscale,
|
||||
@@ -52,16 +51,14 @@ class CutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="cutlass",
|
||||
)
|
||||
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4, getattr(layer, "weights_padding_cols", 0)
|
||||
padded_n=x.shape[-1] + weights_padding_bytes * 2,
|
||||
)
|
||||
|
||||
out = cutlass_scaled_fp4_mm(
|
||||
|
||||
@@ -5,7 +5,6 @@ import torch
|
||||
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
pad_nvfp4_activation_for_cutlass,
|
||||
pad_nvfp4_weight_for_cutlass,
|
||||
slice_nvfp4_output,
|
||||
swizzle_blockscale,
|
||||
@@ -58,16 +57,14 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="flashinfer-cutlass",
|
||||
)
|
||||
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4, getattr(layer, "weights_padding_cols", 0)
|
||||
padded_n=x.shape[-1] + weights_padding_bytes * 2,
|
||||
)
|
||||
|
||||
out = flashinfer_scaled_fp4_mm(
|
||||
@@ -189,16 +186,14 @@ class FlashInferCudnnNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="flashinfer-cudnn",
|
||||
)
|
||||
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4, getattr(layer, "weights_padding_cols", 0)
|
||||
padded_n=x.shape[-1] + weights_padding_bytes * 2,
|
||||
)
|
||||
|
||||
out = flashinfer_scaled_fp4_mm(
|
||||
|
||||
Reference in New Issue
Block a user