[Perf] Optimize cutlass fp8 scaled mm bypassing padding, 20% kernel performance improvement (#43706)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Wentao Ye
2026-06-01 09:05:21 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent bd0aecdc08
commit 985c97a6a8
@@ -312,7 +312,7 @@ class CutlassFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
) -> torch.Tensor:
out_dtype = self.config.out_dtype
if self.is_hopper:
return torch.ops.vllm.padded_cutlass(
return torch.ops.vllm.dynamic_padded_cutlass(
A,
B,
As,
@@ -320,14 +320,14 @@ class CutlassFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
list(self.weight_group_shape),
out_dtype,
)
else:
return ops.cutlass_scaled_mm(
A,
B.T,
out_dtype=out_dtype,
scale_a=As,
scale_b=Bs.T,
)
return ops.cutlass_scaled_mm(
A,
B.T,
out_dtype=out_dtype,
scale_a=As,
scale_b=Bs.T,
)
def cutlass_scaled_mm(
@@ -397,8 +397,56 @@ def _padded_cutlass_fake(
)
def _dynamic_padded_cutlass(
qx: torch.Tensor,
weight: torch.Tensor,
x_scale: torch.Tensor,
weight_scale: torch.Tensor,
block_size: list[int],
output_dtype: torch.dtype,
) -> torch.Tensor:
def run_padded(
qx: torch.Tensor,
weight: torch.Tensor,
x_scale: torch.Tensor,
weight_scale: torch.Tensor,
) -> torch.Tensor:
return _padded_cutlass(
qx, weight, x_scale, weight_scale, block_size, output_dtype
)
def run_direct(
qx: torch.Tensor,
weight: torch.Tensor,
x_scale: torch.Tensor,
weight_scale: torch.Tensor,
) -> torch.Tensor:
return cutlass_scaled_mm(
qx, weight, x_scale, weight_scale, block_size, output_dtype
)
if torch.compiler.is_compiling():
return torch.cond(
qx.shape[0] % 4 != 0,
run_padded,
run_direct,
(qx, weight, x_scale, weight_scale),
)
if qx.shape[0] % 4 != 0:
return run_padded(qx, weight, x_scale, weight_scale)
return run_direct(qx, weight, x_scale, weight_scale)
direct_register_custom_op(
"padded_cutlass",
_padded_cutlass,
fake_impl=_padded_cutlass_fake,
)
direct_register_custom_op(
"dynamic_padded_cutlass",
_dynamic_padded_cutlass,
fake_impl=_padded_cutlass_fake,
)