[Perf] Use upstream CUTLASS for SM90 Block FP8 kernel (#23280)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Michael Goin
2025-09-11 15:43:14 -07:00
committed by GitHub
co-authored by Wentao Ye
parent d4fd2768ef
commit c3aea10dc8
13 changed files with 221 additions and 1260 deletions
+11 -1
View File
@@ -30,7 +30,8 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
register_moe_scaling_factors, rotate_flashinfer_fp8_moe_weights,
select_cutlass_fp8_gemm_impl, swap_w13_to_w31)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
get_col_major_tma_aligned_tensor, requant_weight_ue8m0_inplace)
get_col_major_tma_aligned_tensor, requant_weight_ue8m0_inplace,
should_use_deepgemm_for_fp8_linear)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin,
prepare_moe_fp8_layer_for_marlin)
@@ -462,6 +463,15 @@ class Fp8LinearMethod(LinearMethodBase):
block_sz,
)
# SM90 Block FP8 CUTLASS requires row-major weight scales
if (self.block_quant and current_platform.is_device_capability(90)
and self.cutlass_block_fp8_supported
and not should_use_deepgemm_for_fp8_linear(
torch.bfloat16, layer.weight)):
layer.weight_scale_inv = Parameter(
layer.weight_scale_inv.data.T.contiguous(),
requires_grad=False)
def apply(self,
layer: torch.nn.Module,
x: torch.Tensor,