[CUDA][CUTLASS] Enable cutlass scaled mm for non-compatible sizes (#41868)

Signed-off-by: John Calderon <jcalderon@nvidia.com>
This commit is contained in:
John Calderon
2026-05-08 15:58:05 -07:00
committed by GitHub
parent 6881c754e1
commit 8a2fc80b84
2 changed files with 111 additions and 1 deletions
@@ -40,6 +40,18 @@ MNK_FACTORS = [
(512, 24576, 128),
]
# Shapes with N or K not divisible by 16. These exercise the padding path
# inside CutlassFP8ScaledMMLinearKernel.apply_scaled_mm (e.g. Qwen2.5-VL
# vision MLP dims).
UNALIGNED_MNK_FACTORS = [
(32, 3420, 1280),
(32, 1280, 6840),
(1, 3420, 1280),
(64, 6840, 1280),
(16, 100, 200),
(33, 255, 513),
]
CUDA_DEVICES = [
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
]
@@ -152,6 +164,62 @@ def test_cutlass_fp8_gemm(
cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape, use_bias)
@pytest.mark.parametrize("m,n,k", UNALIGNED_MNK_FACTORS)
@pytest.mark.parametrize(
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize(
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize("use_bias", [True, False])
@pytest.mark.skipif(
not current_platform.has_device_capability(89),
reason="FP8 is not supported on this GPU type.",
)
def test_cutlass_fp8_gemm_padded(
m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool
):
"""Test CUTLASS FP8 GEMM with padding for non-16-aligned N/K dims.
Exercises CutlassFP8ScaledMMLinearKernel.apply_scaled_mm which pads
inputs to satisfy CUTLASS alignment requirements — the path taken by
models like Qwen2.5-VL whose vision MLP has non-16-aligned dims.
"""
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
CutlassFP8ScaledMMLinearKernel,
)
a = to_fp8(torch.randn((m, k), device="cuda"))
b = to_fp8(torch.randn((n, k), device="cuda").t())
a_scales_shape = scale_shape(a.shape, a_scale_group_shape)
b_scales_shape = scale_shape(b.shape, b_scale_group_shape)
scale_a = torch.randn(a_scales_shape, device="cuda", dtype=torch.float32)
scale_b = torch.randn(b_scales_shape, device="cuda", dtype=torch.float32)
scale_a = scale_a.t().contiguous().t()
scale_b = scale_b.t().contiguous().t()
out_dtype = torch.bfloat16
bias = torch.rand((n,), device="cuda", dtype=out_dtype) * 10 if use_bias else None
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
kernel = object.__new__(CutlassFP8ScaledMMLinearKernel)
out = kernel.apply_scaled_mm(
A=a,
B=b,
out_dtype=out_dtype,
As=scale_a,
Bs=scale_b,
bias=bias,
output_shape=[m, n],
)
torch.testing.assert_close(out, baseline, rtol=5e-1, atol=1.5e-1)
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize(
"a_scale_group_shape,b_scale_group_shape", [((1, 128), (128, 128))]