[Perf] Optimize cutlass fp8 scaled mm bypassing padding, 20% kernel performance improvement (#43706)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
bd0aecdc08
commit
985c97a6a8
@@ -312,7 +312,7 @@ class CutlassFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
) -> torch.Tensor:
|
||||
out_dtype = self.config.out_dtype
|
||||
if self.is_hopper:
|
||||
return torch.ops.vllm.padded_cutlass(
|
||||
return torch.ops.vllm.dynamic_padded_cutlass(
|
||||
A,
|
||||
B,
|
||||
As,
|
||||
@@ -320,14 +320,14 @@ class CutlassFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
list(self.weight_group_shape),
|
||||
out_dtype,
|
||||
)
|
||||
else:
|
||||
return ops.cutlass_scaled_mm(
|
||||
A,
|
||||
B.T,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=As,
|
||||
scale_b=Bs.T,
|
||||
)
|
||||
|
||||
return ops.cutlass_scaled_mm(
|
||||
A,
|
||||
B.T,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=As,
|
||||
scale_b=Bs.T,
|
||||
)
|
||||
|
||||
|
||||
def cutlass_scaled_mm(
|
||||
@@ -397,8 +397,56 @@ def _padded_cutlass_fake(
|
||||
)
|
||||
|
||||
|
||||
def _dynamic_padded_cutlass(
|
||||
qx: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
block_size: list[int],
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
def run_padded(
|
||||
qx: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return _padded_cutlass(
|
||||
qx, weight, x_scale, weight_scale, block_size, output_dtype
|
||||
)
|
||||
|
||||
def run_direct(
|
||||
qx: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return cutlass_scaled_mm(
|
||||
qx, weight, x_scale, weight_scale, block_size, output_dtype
|
||||
)
|
||||
|
||||
if torch.compiler.is_compiling():
|
||||
return torch.cond(
|
||||
qx.shape[0] % 4 != 0,
|
||||
run_padded,
|
||||
run_direct,
|
||||
(qx, weight, x_scale, weight_scale),
|
||||
)
|
||||
|
||||
if qx.shape[0] % 4 != 0:
|
||||
return run_padded(qx, weight, x_scale, weight_scale)
|
||||
|
||||
return run_direct(qx, weight, x_scale, weight_scale)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
"padded_cutlass",
|
||||
_padded_cutlass,
|
||||
fake_impl=_padded_cutlass_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
"dynamic_padded_cutlass",
|
||||
_dynamic_padded_cutlass,
|
||||
fake_impl=_padded_cutlass_fake,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user