[Linear] [Kernel] add block-wise scaled_mm (#49932)

Signed-off-by: Zhu, Zufang <zufang.zhu@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
zofia
2026-08-06 14:40:26 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent 8c0f029ac7
commit 8060260e1d
3 changed files with 171 additions and 0 deletions
@@ -3,6 +3,7 @@
# Adapted from https://github.com/sgl-project/sglang/pull/2575
import itertools
import types
import pytest
import torch
@@ -200,6 +201,65 @@ def test_w8a8_block_fp8_cutlass_matmul():
assert rel_diff < 0.001
@pytest.mark.skipif(
not (current_platform.is_cuda() and current_platform.has_device_capability(90)),
reason="torch._scaled_mm DeepSeek-style block scaling only supports SM90.",
)
def test_w8a8_block_fp8_torch_scaled_mm_matmul():
# BlockWiseTorchFP8ScaledMMLinearKernel: 1x128 activation + 128x128 weight
# block scaling routed through torch._scaled_mm. M=83 is not a multiple of
# 4 so this also exercises the M-padding path.
from vllm.model_executor.kernels.linear.scaled_mm.pytorch import (
BlockWiseTorchFP8ScaledMMLinearKernel,
)
M = 83
N = 576
K = 7168
block_size = [128, 128]
out_dtype = torch.bfloat16
seed = 0
torch.manual_seed(seed)
factor_for_scale = 1e-2
fp8_info = torch.finfo(torch.float8_e4m3fn)
fp8_max, fp8_min = fp8_info.max, fp8_info.min
A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
block_n, block_k = block_size[0], block_size[1]
n_tiles = (N + block_n - 1) // block_n
k_tiles = (K + block_k - 1) // block_k
Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale
# Reference uses row-major activation scales.
A_fp8, As = per_token_group_quant_fp8(
A_fp32, block_size[1], column_major_scales=False
)
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
# The kernel expects column-major activation scales on CUDA.
A_fp8_cuda, As_cuda = per_token_group_quant_fp8(
A_fp32, block_size[1], column_major_scales=True
)
stub = BlockWiseTorchFP8ScaledMMLinearKernel.__new__(
BlockWiseTorchFP8ScaledMMLinearKernel
)
stub.config = types.SimpleNamespace(out_dtype=out_dtype)
out = stub.apply_block_scaled_mm(
A_fp8_cuda.cuda(), B_fp8.cuda(), As_cuda.cuda(), Bs.cuda()
)
ref_out = ref_out.cuda()
rel_diff = torch.mean(
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
assert rel_diff < 0.001
@pytest.mark.skipif(
current_platform.is_fp8_fnuz(),
reason="This platform supports e4m3fnuz, not e4m3fn.",
@@ -187,6 +187,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.marlin import (
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.pytorch import (
BlockWiseTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
RowWiseTorchFP8ScaledMMLinearKernel,
@@ -281,6 +282,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
PerTensorTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
RowWiseTorchFP8ScaledMMLinearKernel,
BlockWiseTorchFP8ScaledMMLinearKernel,
},
"aiter": {
AiterInt8ScaledMMLinearKernel,
@@ -380,6 +382,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
MarlinFP8ScaledMMLinearKernel,
TritonFp8BlockScaledMMKernel,
HummingFP8ScaledMMLinearKernel,
BlockWiseTorchFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
AiterFp8BlockScaledMMKernel,
@@ -6,8 +6,11 @@ import math
import torch
from vllm.config import CompilationMode, get_current_vllm_config
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.platforms import current_platform
from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel
from .ScaledMMLinearKernel import (
FP8ScaledMMLinearKernel,
FP8ScaledMMLinearLayerConfig,
@@ -244,3 +247,108 @@ class ChannelWiseTorchFP8ScaledMMLinearKernel(TorchFP8ScaledMMLinearKernel):
if bias is not None:
output = output + bias
return output.to(out_dtype).view(*output_shape)
class BlockWiseTorchFP8ScaledMMLinearKernel(Fp8BlockScaledMMLinearKernel):
"""FP8 block-scaled linear kernel using ``torch._scaled_mm``.
Implements the block-scaled path of ``torch._scaled_mm``, which
dispatches on the shapes of the scale tensors. For ``A = [M, K]`` and
``B = [K, N]`` (both fp8) the op's block path requires, with float32
scales:
* 1x128 activation: ``scale_a = [M, ceil(K / 128)]``
* 128x128 weight: ``scale_b = [ceil(K / 128), ceil(N / 128)]``
"""
def __init__(self, config: FP8ScaledMMLinearLayerConfig) -> None:
super().__init__(config)
act_scale_descriptor = config.activation_quant_key.scale
self.quant_fp8 = QuantFP8(
static=act_scale_descriptor.static,
group_shape=act_scale_descriptor.group_shape,
num_token_padding=self.get_output_padding(),
use_ue8m0=False,
column_major_scales=current_platform.is_cuda_alike(),
)
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not (current_platform.is_cuda_alike() or current_platform.is_xpu()):
return False, "requires CUDA, ROCm or XPU."
# torch._scaled_mm DeepSeek-style (1x128, 128x128) block scaling is
# only implemented for SM90 (Hopper) on NVIDIA CUDA. See PyTorch's
# _check_deepseek_support():
# https://github.com/pytorch/pytorch/blob/33812ece06e2b0d597f73fbe41de03a83f9109f9/aten/src/ATen/native/cuda/ScaledBlas.cpp#L804-L820 # noqa: E501
if current_platform.is_cuda():
is_sm90 = (
compute_capability == 90
if compute_capability is not None
else current_platform.is_device_capability(90)
)
if not is_sm90:
return (
False,
"DeepSeek-style (1x128, 128x128) block scaling on CUDA "
"requires compute capability 90 (Hopper).",
)
return True, None
@classmethod
def can_implement(
cls, config: FP8ScaledMMLinearLayerConfig
) -> tuple[bool, str | None]:
can_implement_base, reason = super().can_implement(config)
if not can_implement_base:
return can_implement_base, reason
act_group_shape = config.activation_quant_key.scale.group_shape
if act_group_shape != GroupShape(1, 128):
return (
False,
"requires 1x128 activation quantization.",
)
weight_group_shape = config.weight_quant_key.scale.group_shape
if weight_group_shape != GroupShape(128, 128):
return (
False,
"requires 128x128 block weight quantization.",
)
return True, None
def apply_block_scaled_mm(
self,
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
) -> torch.Tensor:
# torch._scaled_mm implemented by cuBLASLt requires the M dimension to be a
# multiple of 4; pad A (and its column-major scale) up and slice back.
# https://docs.nvidia.com/cuda/cublas/index.html?highlight=128%2520element%25201D%2520and%2520128x128%25202D#scaling-factors-layouts # noqa: E501
M = A.shape[0]
pad = (-M) % 4 if current_platform.is_cuda() else 0
if pad:
Ap = A.new_empty((M + pad, A.shape[1]))
Ap[:M] = A
A = Ap
# As is column-major [M, ceil(K/128)] (stride (1, M)); build a padded
# column-major buffer and copy only the valid rows. Padded rows are left
# uninitialized — they don't affect the sliced-back output.
Asp = As.new_empty((As.shape[1], M + pad)).t()
Asp[:M] = As
As = Asp
output = torch._scaled_mm(
A,
B.t(),
scale_a=As,
scale_b=Bs.t(),
out_dtype=self.config.out_dtype,
)
if type(output) is tuple and len(output) == 2:
output = output[0]
if pad:
output = output[:M, :]
return output